diff --git a/.editorconfig b/.editorconfig index 3d722d0b..33d7cc66 100644 --- a/.editorconfig +++ b/.editorconfig @@ -30,6 +30,6 @@ indent_style = tab indent_style = space indent_size = 4 -[.gitlab-ci.yml}] +[.gitlab-ci.yml] indent_style = space indent_size = 2 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61fefa97..597241bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,10 +50,10 @@ test:py3.6: - docker except: - tags - - demo build:py2: stage: build + image: python:2.7 script: - /bin/bash tests/run_build.sh tags: @@ -65,6 +65,7 @@ build:py2: paths: - dist/ - meta/ + expire_in: 2 mos build:py3: stage: build @@ -79,18 +80,39 @@ build:py3: paths: - dist/ - meta/ + expire_in: 2 mos + +build:doc: + stage: build + image: python:3.6 + script: + - sed -e "s,^#\(.*\)@AUTOFLASK@\(.*\)$,\1sphinxcontrib.autohttp.flask\2," docs/conf.py + - apt-get update && apt-get install -y unzip + - wget https://bitbucket.org/birkenfeld/sphinx-contrib/get/8e295053a27d.zip + - unzip 8e295053a27d.zip + - (cd birkenfeld-sphinx-contrib-8e295053a27d/httpdomain/ && python setup.py build && python setup.py install) + - pip install -r docs/requirements.txt + - make doc + tags: + - build + only: + - master + artifacts: + paths: + - docs/_build/html + expire_in: 2 mos + allow_failure: true build:docker:latest: stage: build script: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - docker build -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest -f docker/Dockerfile . - - docker tag $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest-py3.6 -# - docker build -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest-py3.6 -f docker/Dockerfile-py3.6 . - - cd docker/demo/docker-pg && docker build -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:latest . + - docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest -f docker/Dockerfile . + - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 .) + - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.0.54 .) - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest - - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest-py3.6 - - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:latest + - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 + - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.0.54 tags: - registry only: @@ -100,28 +122,41 @@ build:docker:release: stage: build script: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - docker build -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG -f docker/Dockerfile . - - docker tag $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG-py3.6 -# - docker build -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG-py3.6 -f docker/Dockerfile-py3.6 . - - cd docker/demo/docker-pg && docker build -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$CI_COMMIT_TAG . + - docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG -f docker/Dockerfile . + - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 .) + - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.0.54 .) - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG - - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG-py3.6 - - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$CI_COMMIT_TAG + - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 + - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.0.54 only: - tags tags: - registry +build:docker:demo: + stage: build + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:demo -f docker/Dockerfile . + - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:demo + - "curl $SENTRY_WEBHOOK -X POST -H 'Content-Type: application/json' -d '{\"version\": \"'$CI_COMMIT_REF_NAME'_'$CI_COMMIT_SHA'\"}'" + only: + - demo + tags: + - registry + deploy:demo: stage: deploy script: - - find docker/demo/ -name "install" | xargs sed -i "s/@build@/$(git rev-parse HEAD)/" + - find docker/demo/ -name "install" -o -name "init" | xargs sed -i "s/@build@/$CI_COMMIT_SHA/" - cd docker/demo/ && find . -maxdepth 1 -type d -a ! -name dist -exec cp -r ../../dist "{}/" \; -exec cp -r ../../meta "{}/" \; && cd ../.. - find docker/demo/ -name "Dockerfile" | xargs sed -i "s,^.*@ARTIFACTS@.*$,COPY dist/*.tar.gz /tmp/burpui.dev.tar.gz,;s,^.*@BUIAGENT_ARTIFACTS@.*$,COPY meta/burp-ui-agent*.tar.gz /tmp/burp-ui-agent.dev.tar.gz," - test -d /srv/demo/docker && rm -rf /srv/demo/docker - cp -r docker/demo/ /srv/demo/docker - cd /srv/demo/docker/ - - docker-compose build +# old docker client, we need the "-e" flag + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN -e $DUMMY_EMAIL $CI_REGISTRY + - docker-compose build --pull - docker-compose stop - docker-compose rm -f - docker-compose up -d diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md index 4521aca5..21a09b1c 100644 --- a/.gitlab/issue_templates/Bug.md +++ b/.gitlab/issue_templates/Bug.md @@ -40,18 +40,22 @@ Unable to login: SQL error ``` $ burp -v -burp-2.0.54 +burp-2.1.18 ``` # Sysinfo ``` $ bui-manage sysinfo -Python version: 3.6.1 -Burp-UI version: 0.5.0 (stable) -Single mode: True -Backend version: 2 -Config file: share/burpui/etc/burpui.sample.cfg +Python version: 3.6.1 +Burp-UI version: 0.5.0 (stable) +Single mode: True +Backend version: 2 +WebSocket embedded: False +WebSocket available: True +Config file: share/burpui/etc/burpui.sample.cfg +Burp client version: 2.1.18 +Burp server version: 2.1.18 ``` # Steps to reproduce diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 753817d0..3ca5d51c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,15 +4,30 @@ Changelog Current ------- -- **BREAKING**: the *BASIC* `ACL` engine will now grant users on all agents if they are not explicitly defined +- **BREAKING**: the *BASIC* ``ACL`` engine will now grant users on all agents if they are not explicitly defined - **BREAKING**: the *Burp1* and *Burp2* configuration sections have been merged into one single *Burp* section +- **BREAKING**: the *running* backups are now displayed in ``green`` instead of ``blue`` +- **BREAKING**: the docker postgresql image was upgraded from 9.6 to 10.1, you'll have to manually upgrade/migrate your data `following this documentation `_ - Add: new plugins system to allow users to write their own modules - Add: `Italian translation `_ thanks to Enrico +- Add: new `client configuration templates `_ - Add: `backups deletion `_ +- Add: `show last client status in client view `_ - Add: `record login failure attempt `_ - Add: `support new burp counters `_ +- Add: `support new burp pair options `_ - Add: `support new reset list (:=) syntax `_ - Add: `new websocket server `_ +- Improvement: `better ACL engine `_ +- Fix: issue `#213 `_ +- Fix: issue `#225 `_ +- Fix: issue `#226 `_ +- Fix: issue `#227 `_ +- Fix: issue `#234 `_ +- Fix: issue `#235 `_ +- Fix: issue `#236 `_ +- Fix: issue `#242 `_ +- Fix: issue `#245 `_ - `Full changelog `__ 0.5.1 (05/26/2017) diff --git a/LICENSE b/LICENSE index 00ed3499..a5ad23ab 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ BSD 3-clause License -Copyright (c) 2014-2017 by Benjamin SANS (Ziirish) http://ziirish.info/ +Copyright (c) 2014-2018 by Benjamin SANS (Ziirish) http://ziirish.info/ All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/Makefile b/Makefile index c220ef45..6a47987a 100644 --- a/Makefile +++ b/Makefile @@ -22,12 +22,13 @@ clean: @find . -type d -name "__pycache__" -exec rm -rf "{}" \; || true @find . -type f -name "*.pyc" -delete || true @rm -rf build dist burp_ui.egg-info docs/_build || true + @cd docs && make clean clean_coverage: @rm -f .coverage flake8: @echo 'Checking pep8 compliance and errors...' - @flake8 --ignore=E501 burpui + @flake8 --ignore=E501,E722 burpui check: pep8 pyflakes doc_coverage test diff --git a/burpui/__main__.py b/burpui/__main__.py index 83f08a41..9e03e197 100644 --- a/burpui/__main__.py +++ b/burpui/__main__.py @@ -141,6 +141,7 @@ def celery(): parser = ArgumentParser('bui-celery') parser.add_argument('-c', '--config', dest='config', help='burp-ui configuration file', metavar='') + parser.add_argument('-t', '--type', dest='type', help='celery mode', metavar='') parser.add_argument('-m', '--mode', dest='mode', help='application mode', metavar='') parser.add_argument('remaining', nargs=REMAINDER) @@ -155,6 +156,11 @@ def celery(): else: conf = lookup_file() + if options.type: + celery_mode = options.type + else: + celery_mode = 'worker' + # make conf path absolute if not conf.startswith('/'): curr = os.getcwd() @@ -169,7 +175,7 @@ def celery(): args = [ 'celery', - 'worker', + celery_mode, '-A', 'worker.celery' ] diff --git a/burpui/agent.py b/burpui/agent.py index ef1a15e5..04b7d588 100644 --- a/burpui/agent.py +++ b/burpui/agent.py @@ -28,6 +28,12 @@ from .utils import BUIlogging from .config import config from .desc import __version__ +try: + from sendfile import sendfile + USE_SENDFILE = True +except ImportError: + USE_SENDFILE = False + G_PORT = 10000 G_BIND = u'::' G_SSL = False @@ -219,11 +225,20 @@ class BUIAgent(BUIbackend, BUIlogging): self.request.sendall(b'OK') self.request.sendall(struct.pack('!Q', size)) with open(path, 'rb') as f: - buf = f.read(1024) - while buf: - self._logger('info', 'sending {} Bytes'.format(len(buf))) - self.request.sendall(buf) - buf = f.read(1024) + if not USE_SENDFILE: + while True: + buf = f.read(1024) + if not buf: + break + self._logger('info', 'sending {} Bytes'.format(len(buf))) + self.request.sendall(buf) + else: + offset = 0 + while True: + sent = sendfile(self.request.fileno(), f.fileno(), offset, size) + if sent == 0: + break + offset += sent os.unlink(path) lengthbuf = self.request.recv(8) length, = struct.unpack('!Q', lengthbuf) diff --git a/burpui/api/__init__.py b/burpui/api/__init__.py index 34ee4c8e..175b31a4 100644 --- a/burpui/api/__init__.py +++ b/burpui/api/__init__.py @@ -14,7 +14,7 @@ import uuid import hashlib import logging -from flask import Blueprint, Response, request, current_app, session +from flask import Blueprint, Response, request, current_app, session, abort from flask_restplus import Api as ApiPlus from flask_login import current_user from importlib import import_module @@ -31,12 +31,17 @@ bui = current_app # type: BUIServer EXEMPT_METHODS = set(['OPTIONS']) +def force_refresh(): + return request.headers.get('X-No-Cache', False) is not False + + def cache_key(): - key = '{}-{}-{}-{}-{}'.format( + key = '{}-{}-{}-{}-{}-{}'.format( session.get('login', uuid.uuid4()), request.path, request.values, request.headers.get('X-Session-Tag', ''), + request.cookies, session.get('language', '') ) key = hashlib.sha256(to_bytes(key)).hexdigest() @@ -61,7 +66,7 @@ def api_login_required(func): not bui.config.get('LOGIN_DISABLED', False)): if not current_user.is_authenticated: if request.headers.get('X-From-UI', False): - return Response('Access denied', 403) + abort(403) return Response( 'Could not verify your access level for that URL.\n' 'You have to login with proper credentials', 401, @@ -70,6 +75,25 @@ def api_login_required(func): return decorated_view +def check_acl(func): + """Custom decorator to check if the ACL are in use or not""" + @wraps(func) + def decorated_view(*args, **kwargs): + if request.method in EXEMPT_METHODS: # pragma: no cover + return func(*args, **kwargs) + # 'func' is a Flask.view.MethodView so we have access to some special + # params + cls = func.view_class + login_required = getattr(cls, 'login_required', True) + if (bui.auth != 'none' and + login_required and + not bui.config.get('LOGIN_DISABLED', False)): + if current_user.is_anonymous: + abort(403) + return func(*args, **kwargs) + return decorated_view + + class Api(ApiPlus): """Wrapper class around :class:`flask_restplus.Api`""" logger = logging.getLogger('burp-ui') @@ -82,8 +106,11 @@ class Api(ApiPlus): def load_all(self): if config['WITH_LIMIT']: - from ..ext.limit import limiter - self.decorators.append(limiter.limit(config['BUI_RATIO'])) + try: + from ..ext.limit import limiter + self.decorators.append(limiter.limit(config['BUI_RATIO'])) + except ImportError: + self.logger.warning('Unable to import limiter module') """hack to automatically import api modules""" if not self.loaded: sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -123,6 +150,18 @@ class Api(ApiPlus): return decorated return decorator + def acl_admin_or_moderator_required(self, message='Access denied', code=403): + def decorator(func): + @wraps(func) + def decorated(resource, *args, **kwargs): + if not current_user.is_anonymous and \ + not current_user.acl.is_admin() and \ + not current_user.acl.is_moderator(): + resource.abort(code, message) + return func(resource, *args, **kwargs) + return decorated + return decorator + def acl_own_or_admin(self, key='name', message='Access denied', code=403): def decorator(func): @wraps(func) diff --git a/burpui/api/admin.py b/burpui/api/admin.py index 149bd093..04707da4 100644 --- a/burpui/api/admin.py +++ b/burpui/api/admin.py @@ -26,6 +26,28 @@ user_fields = ns.model('Users', { 'name': fields.String(required=True, description='User name'), 'backend': fields.String(required=True, description='Backend name'), }) +grant_fields = ns.model('Grants', { + 'id': fields.String(required=True, description='Grant id'), + 'grant': fields.String(required=True, description='Grant content'), + 'backend': fields.String(required=True, description='Backend name'), +}) +group_fields = ns.model('Groups', { + 'id': fields.String(required=True, description='Group id'), + 'grant': fields.String(required=True, description='Group grant content'), + 'members': fields.List(fields.String, required=True, description='Group members'), + 'backend': fields.String(required=True, description='Backend name'), +}) +group_members_fields = ns.model('GroupMembers', { + 'members': fields.List(fields.String, required=True, description='Group members'), + 'grant': fields.String(required=True, description='Group grant content'), +}) +moderator_members_fields = ns.model('ModeratorMembers', { + 'members': fields.List(fields.String, required=True, description='Moderator members'), + 'grant': fields.String(required=True, description='Moderator grant content'), +}) +admin_members_fields = ns.model('AdminMembers', { + 'members': fields.List(fields.String, required=True, description='Admin members'), +}) session_fields = ns.model('Sessions', { 'uuid': fields.String(description='Session id'), 'ip': fields.String(description='IP address'), @@ -52,9 +74,893 @@ class AdminMe(Resource): **GET** method provided by the webservice. - :returns: Users + :returns: User """ - return getattr(current_user, 'real', current_user) + ret = getattr(current_user, 'real', current_user) + return ret + + +@ns.route('/acl/admin/', + '/acl/admin//', + endpoint='acl_admins') +@ns.doc( + params={ + 'backend': 'ACL backend', + 'member': 'Admin member', + } +) +class AclAdmins(Resource): + """The :class:`burpui.api.admin.AclAdmins` resource allows you to + retrieve a list of admins and add/delete them if your + acl backend support those actions. + + This resource is part of the :mod:`burpui.api.admin` module. + """ + parser = ns.parser() + parser.add_argument('memberName', required=False, help='Moderator member', location='values') + + @api.acl_admin_or_moderator_required(message="Not allowed to view admins list") + @ns.marshal_with(admin_members_fields, code=200, description='Success') + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def get(self, backend): + """Returns a list of admin users + + **GET** method provided by the webservice. + + :returns: Members + """ + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(404, "No acl backend '{}' found".format(backend)) + ret = {} + loader = handler.backends[backend] + ret = { + 'members': loader.admins + } + return ret + + @api.acl_admin_required(message="Not allowed to add admin members") + @ns.expect(parser) + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def put(self, backend, member=None): + """Add a member as admin + + **PUT** method provided by the webservice. + """ + args = self.parser.parse_args() + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(404, "No acl backend '{}' found".format(backend)) + + loader = handler.backends[backend] + + member = member or args['memberName'] + + if loader.add_admin is False: + self.abort( + 500, + "The '{}' backend does not support moderator member addition" + "".format(backend) + ) + + success, message, code = loader.add_admin( + member + ) + status = 201 if success else 200 + return [[code, message]], status + + @api.acl_admin_required(message="Not allowed to remove admin members") + @ns.expect(parser) + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def delete(self, backend, member=None): + """Remove an admin member + + **DELETE** method provided by the webservice. + """ + args = self.parser.parse_args() + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(40422, "No acl backend '{}' found".format(backend)) + + loader = handler.backends[backend] + + member = member or args['memberName'] + + if loader.del_admin is False: + self.abort( + 500, + "The '{}' backend does not support admin member deletion" + "".format(backend) + ) + + success, message, code = loader.del_admin( + member + ) + status = 201 if success else 200 + return [[code, message]], status + + +@ns.route('/acl/moderator/', + '/acl/moderator//', + endpoint='acl_moderators') +@ns.doc( + params={ + 'backend': 'ACL backend', + 'member': 'Moderator member', + } +) +class AclModerators(Resource): + """The :class:`burpui.api.admin.AclModerators` resource allows you to + retrieve a list of moderators and add/delete them if your + acl backend support those actions. + + This resource is part of the :mod:`burpui.api.admin` module. + """ + parser = ns.parser() + parser.add_argument('memberName', required=False, help='Moderator member', location='values') + + parser_mod = ns.parser() + parser_mod.add_argument('grants', required=False, help='Moderator grants', location='values') + + @api.acl_admin_or_moderator_required(message="Not allowed to view moderators list") + @ns.marshal_with(moderator_members_fields, code=200, description='Success') + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def get(self, backend): + """Returns a list of moderator users + + **GET** method provided by the webservice. + + :returns: Members + """ + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(404, "No acl backend '{}' found".format(backend)) + ret = {} + loader = handler.backends[backend] + ret = { + 'members': loader.moderators, + 'grant': loader.moderator + } + return ret + + @api.acl_admin_or_moderator_required(message="Not allowed to add moderator members") + @ns.expect(parser) + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def put(self, backend, member=None): + """Add a member as moderator + + **PUT** method provided by the webservice. + """ + args = self.parser.parse_args() + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(404, "No acl backend '{}' found".format(backend)) + + loader = handler.backends[backend] + + member = member or args['memberName'] + + if loader.add_moderator is False: + self.abort( + 500, + "The '{}' backend does not support moderator member addition" + "".format(backend) + ) + + success, message, code = loader.add_moderator( + member + ) + status = 201 if success else 200 + return [[code, message]], status + + @api.acl_admin_or_moderator_required(message="Not allowed to remove moderator members") + @ns.expect(parser) + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def delete(self, backend, member=None): + """Remove a moderator member + + **DELETE** method provided by the webservice. + """ + args = self.parser.parse_args() + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(404, "No acl backend '{}' found".format(backend)) + + loader = handler.backends[backend] + + member = member or args['memberName'] + + if loader.del_moderator is False: + self.abort( + 500, + "The '{}' backend does not support moderator member deletion" + "".format(backend) + ) + + success, message, code = loader.del_moderator( + member + ) + status = 201 if success else 200 + return [[code, message]], status + + @api.acl_admin_or_moderator_required(message="Not allowed to update moderator grants") + @ns.expect(parser_mod) + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def post(self, backend): + """Update moderator grants + + **POST** method provided by the webservice. + """ + args = self.parser.parse_args() + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(404, "No acl backend '{}' found".format(backend)) + + loader = handler.backends[backend] + + grants = args['grants'] + + if loader.mod_moderator is False: + self.abort( + 500, + "The '{}' backend does not support moderator grants edition" + "".format(backend) + ) + + success, message, code = loader.mod_moderator( + grants + ) + status = 201 if success else 200 + return [[code, message]], status + + +@ns.route('/acl/group//', + '/acl/group///', + endpoint='acl_group_members') +@ns.doc( + params={ + 'name': 'Group name', + 'backend': 'ACL backend', + 'member': 'Group member', + } +) +class AclGroup(Resource): + """The :class:`burpui.api.admin.AclGroup` resource allows you to + retrieve a list of members in a given group and add/delete them if your + acl backend support those actions. + + This resource is part of the :mod:`burpui.api.admin` module. + """ + parser = ns.parser() + parser.add_argument('memberName', required=False, help='Group member', location='values') + + @api.acl_admin_or_moderator_required(message="Not allowed to view groups list") + @ns.marshal_with(group_members_fields, code=200, description='Success') + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def get(self, name, backend): + """Returns a list of users in a giver group + + **GET** method provided by the webservice. + + :returns: Members + """ + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(404, "No acl backend '{}' found".format(backend)) + ret = {} + loader = handler.backends[backend] + groups = loader.groups + gname = '@{}'.format(name) + if groups and gname in groups: + ret = { + 'members': groups[gname].get('members', []), + 'grant': groups[gname].get('grants', '') + } + return ret + + @api.acl_admin_or_moderator_required(message="Not allowed to add member in group") + @ns.expect(parser) + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def put(self, name, backend, member=None): + """Add a member in a given group + + **PUT** method provided by the webservice. + """ + args = self.parser.parse_args() + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(404, "No acl backend '{}' found".format(backend)) + + loader = handler.backends[backend] + + member = member or args['memberName'] + + if loader.add_group_member is False: + self.abort( + 500, + "The '{}' backend does not support group member addition" + "".format(backend) + ) + + success, message, code = loader.add_group_member( + name, + member + ) + status = 201 if success else 200 + return [[code, message]], status + + @api.acl_admin_or_moderator_required(message="Not allowed to remove member in group") + @ns.expect(parser) + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def delete(self, name, backend, member=None): + """Remove a member from a given group + + **DELETE** method provided by the webservice. + """ + args = self.parser.parse_args() + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + if backend not in handler.backends: + self.abort(404, "No acl backend '{}' found".format(backend)) + + loader = handler.backends[backend] + + member = member or args['memberName'] + + if loader.del_group_member is False: + self.abort( + 500, + "The '{}' backend does not support group member deletion" + "".format(backend) + ) + + success, message, code = loader.del_group_member( + name, + member + ) + status = 201 if success else 200 + return [[code, message]], status + + +@ns.route('/acl/groups', + '/acl/groups/', + endpoint='acl_groups') +@ns.doc( + params={ + 'name': 'Group name', + } +) +class AclGroups(Resource): + """The :class:`burpui.api.admin.AclGroups` resource allows you to + retrieve a list of groups and to add/update/delete them if your + acl backend support those actions. + + This resource is part of the :mod:`burpui.api.admin` module. + """ + parser_add = ns.parser() + parser_add.add_argument('group', required=True, help='Group name', location='values') + parser_add.add_argument('grant', required=True, help='Group grant content', location='values') + parser_add.add_argument('backend', required=True, help='Backend', location='values') + + parser_mod = ns.parser() + parser_mod.add_argument('grant', required=True, help='Group grant content', location='values') + parser_mod.add_argument('backend', required=True, help='Backend', location='values') + + parser_del = ns.parser() + parser_del.add_argument('backend', required=True, help='Backend', location='values') + + @api.acl_admin_or_moderator_required(message="Not allowed to view groups list") + @ns.marshal_list_with(group_fields, code=200, description='Success') + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def get(self): + """Returns a list of group + + **GET** method provided by the webservice. + + :returns: Groups + """ + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + ret = [] + for _, loader in iteritems(handler.backends): + groups = loader.groups + if groups: + for name, group in iteritems(groups): + ret.append({ + 'id': name.lstrip('@'), + 'grant': group.get('grants', ''), + 'members': group.get('members', []), + 'backend': loader.name + }) + return ret + + @api.disabled_on_demo() + @api.acl_admin_or_moderator_required(message="Not allowed to create groups") + @ns.expect(parser_add) + @ns.doc( + responses={ + 200: 'Request performed with errors', + 201: 'Success', + 403: 'Not allowed', + 400: 'Missing parameters', + 404: 'Backend not found', + 500: 'Backend does not support this operation', + }, + ) + def put(self): + """Create a new group""" + args = self.parser_add.parse_args() + + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0 or \ + args['backend'] not in handler.backends: + self.abort(404, "No acl backend found") + + loader = handler.backends[args['backend']] + + if loader.add_group is False: + self.abort( + 500, + "The '{}' backend does not support group creation" + "".format(args['backend']) + ) + + success, message, code = loader.add_group( + args['group'], + args['grant'] + ) + status = 201 if success else 200 + return [[code, message]], status + + @api.disabled_on_demo() + @api.acl_admin_or_moderator_required(message="Not allowed to delete this group") + @ns.expect(parser_del) + @ns.doc( + responses={ + 200: 'Request performed with errors', + 201: 'Success', + 403: 'Not allowed', + 400: 'Missing parameters', + 404: 'Backend not found', + 500: 'Backend does not support this operation', + }, + ) + def delete(self, name): + """Delete a group""" + args = self.parser_del.parse_args() + + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0 or \ + args['backend'] not in handler.backends: + self.abort(404, "No acl backend found") + + loader = handler.backends[args['backend']] + + if loader.del_group is False: + self.abort( + 500, + "The '{}' backend does not support group deletion" + "".format(args['backend']) + ) + + success, message, code = loader.del_group( + name + ) + status = 201 if success else 200 + return [[code, message]], status + + @api.disabled_on_demo() + @api.acl_admin_or_moderator_required(message="Not allowed to modify this group") + @ns.expect(parser_mod) + @ns.doc( + responses={ + 200: 'Request performed with errors', + 201: 'Success', + 403: 'Not allowed', + 400: 'Missing parameters', + 404: 'Backend not found', + 500: 'Backend does not support this operation', + }, + ) + def post(self, name): + """Change a group""" + args = self.parser_mod.parse_args() + + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0 or \ + args['backend'] not in handler.backends: + self.abort(404, "No acl backend found") + + loader = handler.backends[args['backend']] + + if loader.mod_group is False: + self.abort( + 500, + "The '{}' backend does not support group modification" + "".format(args['backend']) + ) + + success, message, code = loader.mod_group( + name, + args['grant'] + ) + status = 201 if success else 200 + return [[code, message]], status + + +@ns.route('/acl/grants', + '/acl/grants/', + endpoint='acl_grants') +@ns.doc( + params={ + 'name': 'Grant name', + } +) +class AclGrants(Resource): + """The :class:`burpui.api.admin.AclGrants` resource allows you to + retrieve a list of grants and to add/update/delete them if your + acl backend support those actions. + + This resource is part of the :mod:`burpui.api.admin` module. + """ + parser_add = ns.parser() + parser_add.add_argument('grant', required=True, help='Grant name', location='values') + parser_add.add_argument('content', required=True, help='Grant content', location='values') + parser_add.add_argument('backend', required=True, help='Backend', location='values') + + parser_mod = ns.parser() + parser_mod.add_argument('content', required=True, help='Grant content', location='values') + parser_mod.add_argument('backend', required=True, help='Backend', location='values') + + parser_del = ns.parser() + parser_del.add_argument('backend', required=True, help='Backend', location='values') + + @api.acl_admin_or_moderator_required(message="Not allowed to view grants list") + @ns.marshal_list_with(grant_fields, code=200, description='Success') + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def get(self): + """Returns a list of grants + + **GET** method provided by the webservice. + + :returns: Grants + """ + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No acl backend found") + ret = [] + for _, loader in iteritems(handler.backends): + grants = loader.grants + if grants: + for name, grant in iteritems(grants): + ret.append({ + 'id': name, + 'grant': grant, + 'backend': loader.name + }) + return ret + + @api.disabled_on_demo() + @api.acl_admin_or_moderator_required(message="Not allowed to create grants") + @ns.expect(parser_add) + @ns.doc( + responses={ + 200: 'Request performed with errors', + 201: 'Success', + 403: 'Not allowed', + 400: 'Missing parameters', + 404: 'Backend not found', + 500: 'Backend does not support this operation', + }, + ) + def put(self): + """Create a new grant""" + args = self.parser_add.parse_args() + + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0 or \ + args['backend'] not in handler.backends: + self.abort(404, "No acl backend found") + + loader = handler.backends[args['backend']] + + if loader.add_grant is False: + self.abort( + 500, + "The '{}' backend does not support grant creation" + "".format(args['backend']) + ) + + success, message, code = loader.add_grant( + args['grant'], + args['content'] + ) + status = 201 if success else 200 + return [[code, message]], status + + @api.disabled_on_demo() + @api.acl_admin_or_moderator_required(message="Not allowed to delete this grant") + @ns.expect(parser_del) + @ns.doc( + responses={ + 200: 'Request performed with errors', + 201: 'Success', + 403: 'Not allowed', + 400: 'Missing parameters', + 404: 'Backend not found', + 500: 'Backend does not support this operation', + }, + ) + def delete(self, name): + """Delete a grant""" + args = self.parser_del.parse_args() + + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0 or \ + args['backend'] not in handler.backends: + self.abort(404, "No acl backend found") + + loader = handler.backends[args['backend']] + + if loader.del_grant is False: + self.abort( + 500, + "The '{}' backend does not support grant deletion" + "".format(args['backend']) + ) + + success, message, code = loader.del_grant( + name + ) + status = 201 if success else 200 + return [[code, message]], status + + @api.disabled_on_demo() + @api.acl_admin_or_moderator_required(message="Not allowed to modify this grant") + @ns.expect(parser_mod) + @ns.doc( + responses={ + 200: 'Request performed with errors', + 201: 'Success', + 403: 'Not allowed', + 400: 'Missing parameters', + 404: 'Backend not found', + 500: 'Backend does not support this operation', + }, + ) + def post(self, name): + """Change a grant""" + args = self.parser_mod.parse_args() + + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0 or \ + args['backend'] not in handler.backends: + self.abort(404, "No acl backend found") + + loader = handler.backends[args['backend']] + + if loader.mod_grant is False: + self.abort( + 500, + "The '{}' backend does not support grant modification" + "".format(args['backend']) + ) + + success, message, code = loader.mod_grant( + name, + args['content'] + ) + status = 201 if success else 200 + return [[code, message]], status + + +@ns.route('/acl/backends', endpoint='acl_backends') +class AclBackends(Resource): + """The :class:`burpui.api.admin.AclBackends` resource allows you to + retrieve a list of ACL backends with their capabilities. + + This resource is part of the :mod:`burpui.api.admin` module. + """ + backend_fields = ns.model('AclBackends', { + 'name': fields.String(required=True, description='Backend name'), + 'add_grant': fields.Boolean(required=False, default=False, description='Support grant creation'), + 'mod_grant': fields.Boolean(required=False, default=False, description='Support grant edition'), + 'del_grant': fields.Boolean(required=False, default=False, description='Support grant deletion'), + 'add_group': fields.Boolean(required=False, default=False, description='Support group creation'), + 'mod_group': fields.Boolean(required=False, default=False, description='Support group edition'), + 'del_group': fields.Boolean(required=False, default=False, description='Support group deletion'), + 'add_group_member': fields.Boolean(required=False, default=False, description='Support group member addition'), + 'del_group_member': fields.Boolean(required=False, default=False, description='Support group member deletion'), + 'add_moderator': fields.Boolean(required=False, default=False, description='Support moderator creation'), + 'mod_moderator': fields.Boolean(required=False, default=False, description='Support moderator edition'), + 'del_moderator': fields.Boolean(required=False, default=False, description='Support moderator deletion'), + 'add_admin': fields.Boolean(required=False, default=False, description='Support admin creation'), + 'del_admin': fields.Boolean(required=False, default=False, description='Support admin deletion'), + }) + + @api.acl_admin_required(message="Not allowed to view backends list") + @ns.marshal_list_with(backend_fields, code=200, description='Success') + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'No backend found', + }, + ) + def get(self): + """Returns a list of backends + + **GET** method provided by the webservice. + + :returns: Backends + """ + try: + handler = getattr(bui, 'acl_handler') + except AttributeError: + handler = None + + if not handler or len(handler.backends) == 0: + self.abort(404, "No authentication backend found") + ret = [] + for name, backend in iteritems(handler.backends): + back = {} + back['name'] = name + for method in ['add_grant', 'del_grant', 'mod_grant', 'add_group', 'del_group', 'mod_group', 'add_group_member', 'del_group_member', 'add_moderator', 'del_moderator', 'mod_moderator', 'add_admin', 'del_admin']: + back[method] = getattr(backend, method, False) is not False + ret.append(back) + + return ret @ns.route('/auth/users', @@ -85,7 +991,7 @@ class AuthUsers(Resource): parser_del = ns.parser() parser_del.add_argument('backend', required=True, help='Backend', location='values') - @api.acl_admin_required(message="Not allowed to view users list") + @api.acl_admin_or_moderator_required(message="Not allowed to view users list") @ns.marshal_list_with(user_fields, code=200, description='Success') @ns.doc( responses={ @@ -132,7 +1038,7 @@ class AuthUsers(Resource): return ret @api.disabled_on_demo() - @api.acl_admin_required(message="Not allowed to create users") + @api.acl_admin_or_moderator_required(message="Not allowed to create users") @ns.expect(parser_add) @ns.doc( responses={ @@ -174,7 +1080,7 @@ class AuthUsers(Resource): return [[code, message]], status @api.disabled_on_demo() - @api.acl_admin_required(message="Not allowed to delete this user") + @api.acl_admin_or_moderator_required(message="Not allowed to delete this user") @ns.expect(parser_del) @ns.doc( responses={ @@ -314,8 +1220,45 @@ class AuthBackends(Resource): return ret +@ns.route('/session/', + '/session//', + endpoint='other_sessions') +@ns.doc( + params={ + 'user': 'User to get sessions from', + 'id': 'Session id', + } +) +class OtherSessions(Resource): + """The :class:`burpui.api.admin.OtherSessions` resource allows you to + retrieve a list of sessions for a given user. + + This resource is part of the :mod:`burpui.api.admin` module. + """ + + @ns.marshal_list_with(session_fields, code=200, description='Success') + @ns.doc( + responses={ + 403: 'Insufficient permissions', + 404: 'User not found', + }, + ) + def get(self, user=None, id=None): + """Returns a list of sessions + + **GET** method provided by the webservice. + + :returns: Sessions + """ + if id: + return session_manager.get_session_by_id(str(id)) + if not user: + self.abort(404, 'User not found') + return session_manager.get_user_sessions(user) + + @ns.route('/me/session', - '/me/session/', + '/me/session/', endpoint='user_sessions') @ns.doc( params={ @@ -344,7 +1287,7 @@ class MySessions(Resource): :returns: Sessions """ if id: - return session_manager.get_session_by_id(id) + return session_manager.get_session_by_id(str(id)) user = getattr(current_user, 'name', None) if not user: self.abort(404, 'User not found') @@ -369,7 +1312,7 @@ class MySessions(Resource): user = getattr(current_user, 'name', None) if not user: self.abort(404, 'User not found') - store = session_manager.get_session_by_id(id) + store = session_manager.get_session_by_id(str(id)) if not store: self.abort('Session not found') if store.user != user: diff --git a/burpui/api/async.py b/burpui/api/async.py index 95656e3e..2977f4b1 100644 --- a/burpui/api/async.py +++ b/burpui/api/async.py @@ -11,14 +11,13 @@ import os import select import struct -from . import api, cache_key +from . import api, cache_key, force_refresh from .misc import History from .custom import Resource from .client import node_fields -from .clients import RunningBackup, ClientsReport +from .clients import RunningBackup, ClientsReport, RunningClients from ..server import BUIServer # noqa from ..ext.cache import cache -from ..ext.limit import limiter from ..config import config from ..decorators import browser_cache from ..tasks import perform_restore, load_all_tree @@ -26,10 +25,15 @@ from ..tasks import perform_restore, load_all_tree from time import time from zlib import adler32 from flask import url_for, Response, current_app, after_this_request, \ - send_file, redirect + send_file, redirect, request from flask_login import current_user from datetime import timedelta from werkzeug.datastructures import Headers +try: + from .ext.ws import socketio # noqa + WS_AVAILABLE = True +except ImportError: + WS_AVAILABLE = False if config.get('WITH_SQL'): from ..ext.sql import db @@ -53,7 +57,12 @@ class AsyncRestoreStatus(Resource): This resource is part of the :mod:`burpui.api.async` module. """ - decorators = [limiter.exempt] + if config['WITH_LIMIT']: + try: + from ..ext.limit import limiter + decorators = [limiter.exempt] + except ImportError: + pass @ns.doc( responses={ @@ -120,9 +129,14 @@ class AsyncGetFile(Resource): task = perform_restore.AsyncResult(task_id) if task.state != 'SUCCESS': if task.state == 'FAILURE': + err = task.result.get('error') + if err != 'encrypted' and not task.result.get('admin'): + err = 'An error occurred while performing the ' \ + 'restoration. Please contact your administrator ' \ + 'for further details' self.abort( 500, - 'Unsuccessful task: {}'.format(task.result.get('error')) + 'Unsuccessful task:\n{}'.format(err) ) self.abort(400, 'Task not processed yet: {}'.format(task.state)) @@ -154,7 +168,7 @@ class AsyncGetFile(Resource): # ended. Because the fh is open, the file will be actually removed # when the transfer is done and the send_file method has closed # the fh. Only tested on Linux systems. - fh = open(path, 'r') + fh = open(path, 'rb') @after_this_request def remove_file(response): @@ -305,12 +319,15 @@ class AsyncRestore(Resource): strip = args['strip'] fmt = args['format'] or 'zip' passwd = args['pass'] + room = None + if WS_AVAILABLE: + room = request.sid if not files or not name or not backup: self.abort(400, 'missing arguments') # Manage ACL if not current_user.is_anonymous and \ not current_user.acl.is_admin() and \ - not current_user.acl.is_client_allowed(name, server): + not current_user.acl.is_client_rw(name, server): self.abort( 403, 'You are not allowed to perform a restoration for this client' @@ -324,7 +341,9 @@ class AsyncRestore(Resource): fmt, passwd, server, - current_user.name + current_user.name, + not current_user.is_anonymous and current_user.acl.is_admin(), + room ] ) if db: @@ -342,6 +361,61 @@ class AsyncRestore(Resource): return {'id': task.id, 'name': 'perform_restore'}, 202 +@ns.route('/running', + '//running', + '/running/', + '//running/', + endpoint='async_running_clients') +@ns.doc( + params={ + 'server': 'Which server to collect data from when in multi-agent mode', + 'client': 'Client name', + }, +) +class AsyncRunningClients(RunningClients): + """The :class:`burpui.api.async.AsyncRunningClients` resource allows you + to retrieve a list of clients that are currently running a backup. + + This resource is part of the :mod:`burpui.api.async` module. + + This resource is backed by a periodic task. If the periodic task fail or is + not running, we fallback to the "synchronous" API call. + + An optional ``GET`` parameter called ``serverName`` is supported when running + in multi-agent mode. + + .. seealso:: :class:`burpui.api.clients.RunningClients` + """ + + def get(self, client=None, server=None): + """Returns a list of clients currently running a backup + + **GET** method provided by the webservice. + + The *JSON* returned is: + :: + + [ 'client1', 'client2' ] + + + The output is filtered by the :mod:`burpui.misc.acl` module so that you + only see stats about the clients you are authorized to see. + + :param server: Which server to collect data from when in multi-agent mode + :type server: str + + :param client: Ask a specific client in order to know if it is running a backup + :type client: str + + :returns: The *JSON* described above. + """ + server = server or self.parser.parse_args()['serverName'] + res = cache.cache.get('backup_running_result') + if res is None: + res = bui.client.is_one_backup_running(server) + return self._running_clients(res, client, server) + + @ns.route('/backup-running', '//backup-running', endpoint='async_running_backup') @@ -356,12 +430,11 @@ class AsyncRunningBackup(RunningBackup): backup currently. This resource is backed by a periodic task. If the periodic task fail or is - not running, we redirect to the "synchronous" API call. + not running, we fallback to the "synchronous" API call. This resource is part of the :mod:`burpui.api.async` module. """ - @cache.cached(timeout=60, key_prefix=cache_key) @ns.marshal_with( RunningBackup.running_fields, code=200, @@ -392,10 +465,7 @@ class AsyncRunningBackup(RunningBackup): """ res = cache.cache.get('backup_running_result') if res is None: - # redirect to synchronous API call - # FIXME: Since we subclass the original code, we don't need the - # redirect anymore if the redirection is problematic - return redirect(url_for('api.running_backup', server=server)) + res = bui.client.is_one_backup_running(server) return {'running': self._is_one_backup_running(res, server)} @@ -456,7 +526,7 @@ class AsyncHistory(History): """ - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_with( History.history_fields, code=200, @@ -548,7 +618,7 @@ class AsyncClientsReport(ClientsReport): running in multi-agent mode. """ - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_with( ClientsReport.report_fields, code=200, diff --git a/burpui/api/backup.py b/burpui/api/backup.py index 2aa21046..54b22be3 100644 --- a/burpui/api/backup.py +++ b/burpui/api/backup.py @@ -100,7 +100,8 @@ class ServerBackup(Resource): # Manage ACL if not current_user.is_anonymous and \ not current_user.acl.is_admin() and \ - not current_user.acl.is_client_allowed(name, server): + not current_user.acl.is_moderator() and \ + not current_user.acl.is_client_rw(name, server): self.abort(403, 'You are not allowed to cancel a backup for this client') try: return bui.client.cancel_server_backup(name, server) @@ -136,7 +137,8 @@ class ServerBackup(Resource): # Manage ACL if not current_user.is_anonymous and \ not current_user.acl.is_admin() and \ - not current_user.acl.is_client_allowed(name, server): + not current_user.acl.is_moderator() and \ + not current_user.acl.is_client_rw(name, server): self.abort( 403, 'You are not allowed to schedule a backup for this client' diff --git a/burpui/api/client.py b/burpui/api/client.py index e79cf7bf..93f20a36 100644 --- a/burpui/api/client.py +++ b/burpui/api/client.py @@ -8,18 +8,20 @@ """ import os +import re -from . import api, cache_key +from . import api, cache_key, force_refresh from ..server import BUIServer # noqa from .custom import fields, Resource from .custom.inputs import boolean +from ..decorators import browser_cache from ..ext.cache import cache from ..exceptions import BUIserverException from ..utils import NOTIF_ERROR from six import iteritems from flask_restplus.marshalling import marshal -from flask import current_app +from flask import current_app, request from flask_login import current_user bui = current_app # type: BUIServer @@ -154,8 +156,16 @@ class ClientTree(Resource): required=False, default=False ) + parser.add_argument( + 'init', + type=boolean, + help='First call to load the root of the tree', + nullable=True, + required=False, + default=False + ) - @cache.cached(timeout=3600, key_prefix=cache_key) + @cache.cached(timeout=3600, key_prefix=cache_key, unless=force_refresh) @ns.marshal_list_with(node_fields, code=200, description='Success') @ns.expect(parser) @ns.doc( @@ -164,6 +174,7 @@ class ClientTree(Resource): '500': 'Internal failure', }, ) + @browser_cache(3600) def get(self, server=None, name=None, backup=None): """Returns a list of 'nodes' under a given path @@ -224,6 +235,20 @@ class ClientTree(Resource): not current_user.acl.is_client_allowed(name, server): self.abort(403, 'Sorry, you are not allowed to view this client') + from_cookie = None + if args['init'] and not root_list: + from_cookie = request.cookies.get('fancytree-1-expanded', '') + if from_cookie: + args['recursive'] = True + _root = bui.client.get_tree(name, backup, agent=server) + root_list = [x['name'] for x in _root] + for path in from_cookie.split('~'): + if not path.endswith('/'): + path += '/' + if path not in root_list: + root_list.append(path) + root_list = sorted(root_list) + try: root_list_clean = [] @@ -357,7 +382,7 @@ class ClientTreeAll(Resource): help='Which server to collect data from when in multi-agent mode' ) - @cache.cached(timeout=3600, key_prefix=cache_key) + @cache.cached(timeout=3600, key_prefix=cache_key, unless=force_refresh) @ns.marshal_list_with(node_fields, code=200, description='Success') @ns.expect(parser) @ns.doc( @@ -367,6 +392,7 @@ class ClientTreeAll(Resource): '500': 'Internal failure', }, ) + @browser_cache(3600) def get(self, server=None, name=None, backup=None): """Returns a list of all 'nodes' of a given backup @@ -625,7 +651,7 @@ class ClientReport(Resource): ), }) - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_with(report_fields, code=200, description='Success') @ns.expect(parser) @ns.doc( @@ -634,6 +660,7 @@ class ClientReport(Resource): '500': 'Internal failure', }, ) + @browser_cache(1800) def get(self, server=None, name=None, backup=None): """Returns a global report of a given backup/client @@ -854,7 +881,9 @@ class ClientReport(Resource): if not current_user.is_anonymous and \ not current_user.acl.is_admin() and \ - not current_user.acl.is_client_allowed(name, server): + (not current_user.acl.is_moderator() or + current_user.acl.is_moderator() and + not current_user.acl.is_client_rw(name, server)): self.abort(403, 'You don\'t have rights on this client') msg = bui.client.delete_backup(name, backup, server) @@ -906,7 +935,7 @@ class ClientStats(Resource): ), }) - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_list_with(client_fields, code=200, description='Success') @ns.expect(parser) @ns.doc( @@ -915,6 +944,7 @@ class ClientStats(Resource): '500': 'Internal failure', }, ) + @browser_cache(1800) def get(self, server=None, name=None): """Returns a list of backups for a given client @@ -956,3 +986,195 @@ class ClientStats(Resource): except BUIserverException as exp: self.abort(500, str(exp)) return json + + +@ns.route('/labels/', + '//labels/', + endpoint='client_labels') +@ns.doc( + params={ + 'server': 'Which server to collect data from when in multi-agent' + + ' mode', + 'name': 'Client name', + }, +) +class ClientLabels(Resource): + """The :class:`burpui.api.client.ClientLabels` resource allows you to + retrieve the labels of a given client. + + This resource is part of the :mod:`burpui.api.client` module. + + An optional ``GET`` parameter called ``serverName`` is supported when + running in multi-agent mode. + """ + parser = ns.parser() + parser.add_argument( + 'serverName', + help='Which server to collect data from when in multi-agent mode' + ) + parser.add_argument('clientName', help='Client name') + labels_fields = ns.model('ClientLabels', { + 'labels': fields.List(fields.String, description='List of labels'), + }) + + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) + @ns.marshal_list_with(labels_fields, code=200, description='Success') + @ns.expect(parser) + @ns.doc( + responses={ + '403': 'Insufficient permissions', + '500': 'Internal failure', + }, + ) + @browser_cache(1800) + def get(self, server=None, name=None): + """Returns the labels of a given client + + **GET** method provided by the webservice. + + The *JSON* returned is: + :: + + { + "labels": [ + "label1", + "label2" + ] + } + + The output is filtered by the :mod:`burpui.misc.acl` module so that you + only see stats about the clients you are authorized to. + + :param server: Which server to collect data from when in multi-agent + mode + :type server: str + + :param name: The client we are working on + :type name: str + + :returns: The *JSON* described above. + """ + try: + if not current_user.is_anonymous and \ + not current_user.acl.is_admin() and \ + not current_user.acl.is_client_allowed(name, server): + self.abort(403, 'Sorry, you cannot access this client') + labels = self._get_labels(name, server) + except BUIserverException as exp: + self.abort(500, str(exp)) + return {'labels': labels} + + @staticmethod + def _get_labels(client, server): + key = 'labels-{}-{}'.format(client, server) + ret = cache.cache.get(key) + if ret is not None: + return ret + labels = bui.client.get_client_labels(client, agent=server) + ret = [] + for label in labels: + if bui.ignore_labels and \ + re.search('|'.join(bui.ignore_labels), label): + continue + tmp_label = label + if bui.format_labels: + for regex, replace in bui.format_labels: + tmp_label = re.sub(regex, replace, tmp_label) + ret.append(tmp_label) + cache.cache.set(key, ret, 1800) + return ret + + +@ns.route('/running', + '/running/', + '//running', + '//running/', + endpoint='client_running_status') +@ns.doc( + params={ + 'server': 'Which server to collect data from when in multi-agent' + + ' mode', + 'name': 'Client name', + }, +) +class ClientRunningStatus(Resource): + """The :class:`burpui.api.client.ClientRunningStatus` resource allows you to + retrieve the running status of a given client. + + This resource is part of the :mod:`burpui.api.client` module. + + An optional ``GET`` parameter called ``serverName`` is supported when + running in multi-agent mode. + """ + parser = ns.parser() + parser.add_argument( + 'serverName', + help='Which server to collect data from when in multi-agent mode' + ) + parser.add_argument('clientName', help='Client name') + running_fields = ns.model('ClientRunningStatus', { + 'state': fields.LocalizedString(required=True, description='Running state'), + 'percent': fields.Integer( + required=False, + description='Backup progress in percent', + default=-1 + ), + 'phase': fields.String( + required=False, + description='Backup phase', + default=None + ), + 'last': fields.DateTime( + required=False, + dt_format='iso8601', + description='Date of last backup' + ), + }) + + @ns.marshal_list_with(running_fields, code=200, description='Success') + @ns.expect(parser) + @ns.doc( + responses={ + '403': 'Insufficient permissions', + '500': 'Internal failure', + }, + ) + def get(self, server=None, name=None): + """Returns the running status of a given client + + **GET** method provided by the webservice. + + The *JSON* returned is: + :: + + { + "state": "running", + "percent": 42, + "phase": "2", + "last": "now" + } + + The output is filtered by the :mod:`burpui.misc.acl` module so that you + only see stats about the clients you are authorized to. + + :param server: Which server to collect data from when in multi-agent + mode + :type server: str + + :param name: The client we are working on + :type name: str + + :returns: The *JSON* described above. + """ + args = self.parser.parse_args() + server = server or args['serverName'] + name = name or args['clientName'] + try: + if not current_user.is_anonymous and \ + not current_user.acl.is_admin() and \ + not current_user.acl.is_client_allowed(name, server): + self.abort(403, 'Sorry, you cannot access this client') + json = bui.client.get_client_status(name, agent=server) + except BUIserverException as exp: + self.abort(500, str(exp)) + return json diff --git a/burpui/api/clients.py b/burpui/api/clients.py index 48097330..2497c653 100644 --- a/burpui/api/clients.py +++ b/burpui/api/clients.py @@ -7,9 +7,10 @@ .. moduleauthor:: Ziirish """ -from . import api, cache_key +from . import api, cache_key, force_refresh from ..server import BUIServer # noqa from .custom import fields, Resource +from .client import ClientLabels from ..ext.cache import cache from ..exceptions import BUIserverException from ..decorators import browser_cache @@ -22,8 +23,6 @@ bui = current_app # type: BUIServer ns = api.namespace('clients', 'Clients methods') -# Seem to not be used anymore -# TODO: we can probably remove this someday @ns.route('/running', '//running', '/running/', @@ -71,32 +70,40 @@ class RunningClients(Resource): :returns: The *JSON* described above. """ server = server or self.parser.parse_args()['serverName'] + return self._running_clients(None, client, server) + + def _running_clients(self, res, client, server): if client: if not current_user.is_anonymous and \ not current_user.acl.is_admin() and \ not current_user.acl.is_client_allowed(client, server): - running = [] - return running - if bui.client.is_backup_running(client, server): - running = [client] - return running - else: - running = [] - return running + return [] - running = bui.client.is_one_backup_running(server) + if bui.client.is_backup_running(client, server): + return [client] + else: + return [] + + running = res or bui.client.is_one_backup_running(server) # Manage ACL if not current_user.is_anonymous and not current_user.acl.is_admin(): if isinstance(running, dict): - new = {} - for serv in bui.client.servers: + ret = {} + + def __extract_running_clients(serv): try: clients = [x['name'] for x in bui.client.get_all_clients(serv)] except BUIserverException: clients = [] allowed = [x for x in clients if current_user.acl.is_client_allowed(x, serv)] - new[serv] = [x for x in running[serv] if x in allowed] - running = new + return [x for x in running[serv] if x in allowed] + + if server: + return __extract_running_clients(server) + + for serv in bui.client.servers: + ret[serv] = __extract_running_clients(serv) + return ret else: try: clients = [x['name'] for x in bui.client.get_all_clients(server)] @@ -104,6 +111,8 @@ class RunningClients(Resource): clients = [] allowed = [x for x in clients if current_user.acl.is_client_allowed(x, server)] running = [x for x in running if x in allowed] + elif server and isinstance(running, dict): + return running.get(server, []) return running @@ -126,7 +135,6 @@ class RunningBackup(Resource): 'running': fields.Boolean(required=True, description='Is there a backup running right now'), }) - @cache.cached(timeout=60, key_prefix=cache_key) @ns.marshal_with(running_fields, code=200, description='Success') def get(self, server=None): """Tells if a backup is running right now @@ -235,7 +243,7 @@ class ClientsReport(Resource): 'clients': fields.Nested(client_fields, as_list=True, required=True), }) - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_with(report_fields, code=200, description='Success') @ns.expect(parser) @ns.doc( @@ -244,6 +252,7 @@ class ClientsReport(Resource): 500: 'Internal failure', }, ) + @browser_cache(1800) def get(self, server=None): """Returns a global report about all the clients of a given server @@ -414,14 +423,14 @@ class ClientsStats(Resource): parser.add_argument('serverName', help='Which server to collect data from when in multi-agent mode') client_fields = ns.model('ClientsStatsSingle', { 'last': fields.DateTime(required=True, dt_format='iso8601', description='Date of last backup'), - 'human': fields.DateTimeHuman(required=True, attribute='last', description='Human readable date of the last backup'), 'name': fields.String(required=True, description='Client name'), 'state': fields.LocalizedString(required=True, description='Current state of the client (idle, backup, etc.)'), 'phase': fields.String(description='Phase of the current running backup'), 'percent': fields.Integer(description='Percentage done', default=0), + 'labels': fields.List(fields.String, description='List of labels'), }) - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_list_with(client_fields, code=200, description='Success') @ns.expect(parser) @ns.doc( @@ -430,6 +439,7 @@ class ClientsStats(Resource): 500: 'Internal failure', }, ) + @browser_cache(1800) def get(self, server=None): """Returns a list of clients with their states @@ -446,6 +456,7 @@ class ClientsStats(Resource): "state": "idle", "phase": "phase1", "percent": 12, + "labels": [] }, { "last": "never", @@ -453,6 +464,7 @@ class ClientsStats(Resource): "state": "idle", "phase": "phase2", "percent": 42, + "labels": [] } ] } @@ -478,7 +490,17 @@ class ClientsStats(Resource): jso = [x for x in jso if current_user.acl.is_client_allowed(x['name'], server)] except BUIserverException as e: self.abort(500, str(e)) - return jso + ret = [] + for client in jso: + tmp_client = client + try: + labels = ClientLabels._get_labels(client['name'], server) + except BUIserverException as exp: + self.abort(500, str(exp)) + tmp_client['labels'] = labels + ret.append(tmp_client) + + return ret @ns.route('/all', @@ -504,7 +526,7 @@ class AllClients(Resource): 'agent': fields.String(required=False, default=None, description='Associated Agent name'), }) - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_list_with(client_fields, code=200, description='Success') @ns.expect(parser) @ns.doc( @@ -553,17 +575,19 @@ class AllClients(Resource): server = server or args['serverName'] is_admin = current_user.is_anonymous or current_user.acl.is_admin() + is_moderator = current_user.is_anonymous or current_user.acl.is_moderator() user = (args.get('user', current_user.name) or current_user.name) if \ - is_admin \ + is_admin or is_moderator \ else current_user.name # drop privileges when switching user if user != current_user.name: is_admin = False + is_moderator = False if (server and - not is_admin and + not is_admin and not is_moderator and not current_user.acl.is_server_allowed(server)): self.abort(403, "You are not allowed to view this server infos") diff --git a/burpui/api/misc.py b/burpui/api/misc.py index 4eeb6ae7..580756fd 100644 --- a/burpui/api/misc.py +++ b/burpui/api/misc.py @@ -7,7 +7,7 @@ .. moduleauthor:: Ziirish """ -from . import api, cache_key +from . import api, cache_key, force_refresh from ..server import BUIServer # noqa from .custom import fields, Resource from ..exceptions import BUIserverException @@ -242,6 +242,10 @@ class Live(Resource): res.append(data) else: for client in running: + # ACL + if has_acl and not is_admin and \ + not current_user.acl.is_client_allowed(client, server): + continue data = {} data['client'] = client try: @@ -307,7 +311,7 @@ class Languages(Resource): '*': wild, }) - @cache.cached(timeout=3600, key_prefix=cache_key) + @cache.cached(timeout=3600, key_prefix=cache_key, unless=force_refresh) @ns.marshal_with(languages_fields, code=200, description='Success') @browser_cache(3600) def get(self): @@ -361,7 +365,7 @@ class About(Resource): 'burp': fields.Nested(burp_fields, as_list=True, description='Burp version'), }) - @cache.cached(timeout=3600, key_prefix=cache_key) + @cache.cached(timeout=3600, key_prefix=cache_key, unless=force_refresh) @ns.marshal_with(about_fields, code=200, description='Success') @ns.expect(parser) @browser_cache(3600) @@ -467,7 +471,7 @@ class History(Resource): 'name': fields.String(description='Feed name'), }) - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_list_with(history_fields, code=200, description='Success') @ns.expect(parser) @ns.doc( @@ -782,3 +786,13 @@ class History(Resource): ret.append(ev) return ret + + +# @ns.route('/testacl', +# '/testacl/', +# '//testacl', +# '//testacl/', +# endpoint='testacl') +# class TestAcl(Resource): +# def get(self, client=None, server=None): +# return current_user.acl.is_client_rw(client, server) diff --git a/burpui/api/restore.py b/burpui/api/restore.py index 5fed0ad0..7f34ed97 100644 --- a/burpui/api/restore.py +++ b/burpui/api/restore.py @@ -85,18 +85,18 @@ class Restore(Resource): :returns: A :mod:`flask.Response` object representing an archive of the restored files """ args = self.parser.parse_args() - l = args['list'] - s = args['strip'] - f = args['format'] or 'zip' - p = args['pass'] + lst = args['list'] + stp = args['strip'] + fmt = args['format'] or 'zip' + pwd = args['pass'] resp = None # Check params - if not l or not name or not backup: + if not lst or not name or not backup: self.abort(400, 'missing arguments') # Manage ACL if not current_user.is_anonymous and \ not current_user.acl.is_admin() and \ - not current_user.acl.is_client_allowed(name, server): + not current_user.acl.is_client_rw(name, server): self.abort(403, 'You are not allowed to perform a restoration for this client') if server: filename = 'restoration_%d_%s_on_%s_at_%s.%s' % ( @@ -104,19 +104,25 @@ class Restore(Resource): name, server, strftime("%Y-%m-%d_%H_%M_%S", gmtime()), - f) + fmt) else: filename = 'restoration_%d_%s_at_%s.%s' % ( backup, name, strftime("%Y-%m-%d_%H_%M_%S", gmtime()), - f) + fmt) - archive, err = bui.client.restore_files(name, backup, l, s, f, p, server) + archive, err = bui.client.restore_files(name, backup, lst, stp, fmt, pwd, server) if not archive: if err: + if (not current_user.is_anonymous and + not current_user.acl.is_admin() or + bui.demo) and err != 'encrypted': + err = 'An error occurred while performing the ' \ + 'restoration. Please contact your administrator ' \ + 'for further details' return make_response(err, 500) - self.abort(500) + return make_response(err, 500) if not server: try: @@ -126,7 +132,7 @@ class Restore(Resource): # ended. Because the fh is open, the file will be actually removed # when the transfer is done and the send_file method has closed # the fh. Only tested on Linux systems. - fh = open(archive, 'r') + fh = open(archive, 'rb') @after_this_request def remove_file(response): @@ -142,9 +148,9 @@ class Restore(Resource): attachment_filename=filename, mimetype='application/zip') resp.set_cookie('fileDownload', 'true') - except Exception as e: - bui.client.logger.error(str(e)) - self.abort(500, str(e)) + except Exception as exp: + bui.client.logger.error(str(exp)) + self.abort(500, str(exp)) else: # Multi-agent mode try: @@ -157,25 +163,25 @@ class Restore(Resource): bui.client.logger.debug('Need to get {} Bytes : {}'.format(length, socket)) - def stream_file(sock, l): + def stream_file(sock, size): """The restoration took place on another server so we need to stream the file that is not present on the current machine. """ bsize = 1024 received = 0 - if l < bsize: - bsize = l - while received < l: + if size < bsize: + bsize = size + while received < size: buf = b'' - r, _, _ = select.select([sock], [], [], 5) - if not r: + read, _, _ = select.select([sock], [], [], 5) + if not read: raise Exception('Socket timed-out') buf += sock.recv(bsize) if not buf: continue received += len(buf) - self.logger.debug('{}/{}'.format(received, l)) + self.logger.debug('{}/{}'.format(received, size)) yield buf sock.sendall(struct.pack('!Q', 2)) sock.sendall(b'RE') @@ -196,11 +202,11 @@ class Restore(Resource): time(), length, adler32(filename.encode('utf-8')) & 0xffffffff)) - except HTTPException as e: - raise e - except Exception as e: - bui.client.logger.error(str(e)) - self.abort(500, str(e)) + except HTTPException as exp: + raise exp + except Exception as exp: + bui.client.logger.error(str(exp)) + self.abort(500, str(exp)) return resp @@ -298,7 +304,7 @@ class ServerRestore(Resource): # Manage ACL if not current_user.is_anonymous and \ not current_user.acl.is_admin() and \ - not current_user.acl.is_client_allowed(name, server): + not current_user.acl.is_client_rw(name, server): self.abort(403, 'You are not allowed to edit a restoration for this client') try: return bui.client.is_server_restore(name, server) @@ -332,7 +338,7 @@ class ServerRestore(Resource): # Manage ACL if not current_user.is_anonymous and \ not current_user.acl.is_admin() and \ - not current_user.acl.is_client_allowed(name, server): + not current_user.acl.is_client_rw(name, server): self.abort(403, 'You are not allowed to cancel a restoration for this client') try: return bui.client.cancel_server_restore(name, server) @@ -420,7 +426,7 @@ class DoServerRestore(Resource): # Manage ACL if not current_user.is_anonymous and \ not current_user.acl.is_admin() and \ - not current_user.acl.is_client_allowed(name, server) and \ + not current_user.acl.is_client_rw(name, server) and \ not current_user.acl.is_client_allowed(to, server): self.abort( 403, diff --git a/burpui/api/servers.py b/burpui/api/servers.py index e8dfec6e..cb43e7eb 100644 --- a/burpui/api/servers.py +++ b/burpui/api/servers.py @@ -1,10 +1,11 @@ # -*- coding: utf8 -*- # This is a submodule we can also use "from ..api import api" -from . import api, cache_key +from . import api, cache_key, force_refresh from ..server import BUIServer # noqa from .custom import fields, Resource from ..ext.cache import cache +from ..decorators import browser_cache from ..exceptions import BUIserverException from flask import current_app @@ -28,13 +29,14 @@ class ServersStats(Resource): 'name': fields.String(required=True, description='Server name'), }) - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_list_with(servers_fields, code=200, description='Success') @ns.doc( responses={ 500: 'Internal failure', }, ) + @browser_cache(1800) def get(self): """Returns a list of servers (agents) with basic stats @@ -126,7 +128,7 @@ class ServersReport(Resource): 'servers': fields.Nested(server_fields, as_list=True, required=True), }) - @cache.cached(timeout=1800, key_prefix=cache_key) + @cache.cached(timeout=1800, key_prefix=cache_key, unless=force_refresh) @ns.marshal_with(report_fields, code=200, description='Success') @ns.doc( responses={ @@ -134,6 +136,7 @@ class ServersReport(Resource): 500: 'Internal failure', }, ) + @browser_cache(1800) def get(self): """Returns a global report about all the servers managed by Burp-UI diff --git a/burpui/api/settings.py b/burpui/api/settings.py index 154f70ce..b3277f41 100644 --- a/burpui/api/settings.py +++ b/burpui/api/settings.py @@ -17,7 +17,7 @@ from ..utils import NOTIF_INFO from six import iteritems from flask_babel import gettext as _, refresh -from flask import jsonify, request, url_for, current_app +from flask import jsonify, request, url_for, current_app, g from ..datastructures import ImmutableMultiDict bui = current_app # type: BUIServer @@ -56,6 +56,7 @@ class ServerSettings(Resource): noti = bui.client.store_conf_srv(request.form, conf, server) return {'notif': noti}, 200 + @api.disabled_on_demo() @api.acl_admin_required(message='Sorry, you don\'t have rights to access the setting panel') @ns.doc( responses={ @@ -64,6 +65,23 @@ class ServerSettings(Resource): 500: 'Internal failure', } ) + def delete(self, conf=None, server=None): + """Deletes a configuration file""" + try: + conf = unquote(conf) + except: + pass + parser = bui.client.get_parser(agent=server) + return parser.remove_conf(conf) + + @api.acl_admin_or_moderator_required(message='Sorry, you don\'t have rights to access the setting panel') + @ns.doc( + responses={ + 200: 'Success', + 403: 'Insufficient permissions', + 500: 'Internal failure', + } + ) def get(self, conf=None, server=None): """Reads the server configuration @@ -168,6 +186,30 @@ class ServerSettings(Resource): ] }, { "...": "..." } + ], + "hierarchy": [ + { + "children": [ + { + "children": [], + "dir": "/tmp/burp/conf.d", + "full": "/tmp/burp/conf.d/empty.conf", + "name": "empty.conf", + "parent": "/tmp/burp/burp-server.conf" + }, + { + "children": [], + "dir": "/tmp/burp/conf.d", + "full": "/tmp/burp/conf.d/ipv4.conf", + "name": "ipv4.conf", + "parent": "/tmp/burp/burp-server.conf" + } + ], + "dir": "/tmp/burp", + "full": "/tmp/burp/burp-server.conf", + "name": "burp-server.conf", + "parent": null + } ] }, "server_doc": { @@ -212,21 +254,44 @@ class ServerSettings(Resource): res = bui.client.read_conf_srv(conf, server) refresh() # Translate the doc and placeholder API side - doc = bui.client.get_parser_attr('doc', server).copy() - placeholders = bui.client.get_parser_attr('placeholders', server).copy() - for key, val in iteritems(doc): - doc[key] = _(val) - for key, val in iteritems(placeholders): - placeholders[key] = _(val) + cache_keys = { + 'doc': '_doc_parser_{}-{}'.format(server, g.locale), + 'placeholders': '_placeholders_parser_{}-{}'.format(server, g.locale), + 'boolean_srv': '_boolean_srv_parser_{}'.format(server), + 'string_srv': '_string_srv_parser_{}'.format(server), + 'integer_srv': '_integer_srv_parser_{}'.format(server), + 'multi_srv': '_multi_srv_parser_{}'.format(server), + 'values': '_suggest_parser_{}'.format(server), + 'defaults': '_defaults_parser_{}'.format(server), + 'advanced_type': '_advanced_parser_{}'.format(server), + 'pair_associations': '_pair_associations_parser_{}'.format(server), + } + cache_results = {} + for name, key in iteritems(cache_keys): + if not cache.cache.has(key): + if name in ['doc', 'placeholders']: + _tmp = bui.client.get_parser_attr(name, server).copy() + _tmp2 = {} + for k, v in iteritems(_tmp): + _tmp2[k] = _(v) + cache_results[name] = _tmp2 + else: + cache_results[name] = bui.client.get_parser_attr(name, server) + cache.cache.set(key, cache_results[name], 3600) + else: + cache_results[name] = cache.cache.get(key) + return jsonify(results=res, - boolean=bui.client.get_parser_attr('boolean_srv', server), - string=bui.client.get_parser_attr('string_srv', server), - integer=bui.client.get_parser_attr('integer_srv', server), - multi=bui.client.get_parser_attr('multi_srv', server), - server_doc=doc, - suggest=bui.client.get_parser_attr('values', server), - placeholders=placeholders, - defaults=bui.client.get_parser_attr('defaults', server)) + boolean=cache_results['boolean_srv'], + string=cache_results['string_srv'], + integer=cache_results['integer_srv'], + multi=cache_results['multi_srv'], + pair=cache_results['pair_associations'], + advanced=cache_results['advanced_type'], + server_doc=cache_results['doc'], + suggest=cache_results['values'], + placeholders=cache_results['placeholders'], + defaults=cache_results['defaults']) @ns.route('/clients', @@ -239,7 +304,7 @@ class ServerSettings(Resource): ) class ClientsList(Resource): - @api.acl_admin_required(message='Sorry, you don\'t have rights to access the setting panel') + @api.acl_admin_or_moderator_required(message='Sorry, you don\'t have rights to access the setting panel') @ns.doc( responses={ 200: 'Success', @@ -249,10 +314,84 @@ class ClientsList(Resource): ) def get(self, server=None): """Returns a list of clients""" - res = bui.client.clients_list(server) + parser = bui.client.get_parser(agent=server) + res = parser.list_clients() return jsonify(result=res) +@ns.route('/templates', + '//templates', + endpoint='templates_list') +@ns.doc( + params={ + 'server': 'Which server to collect data from when in multi-agent mode', + }, +) +class TemplatesList(Resource): + + @api.acl_admin_or_moderator_required(message='Sorry, you don\'t have rights to access the setting panel') + @ns.doc( + responses={ + 200: 'Success', + 403: 'Insufficient permissions', + 500: 'Internal failure', + } + ) + def get(self, server=None): + """Returns a list of clients""" + parser = bui.client.get_parser(agent=server) + res = parser.list_templates() + return jsonify(result=res) + + +@ns.route('/template', + '//template', + endpoint='new_template', + methods=['PUT']) +@ns.doc( + params={ + 'server': 'Which server to collect data from when in multi-agent mode', + }, +) +class NewTemplateSettings(Resource): + parser = ns.parser() + parser.add_argument('newtemplate', required=True, help="No 'newclient' provided") + + @api.disabled_on_demo() + @api.acl_admin_or_moderator_required(message='Sorry, you don\'t have rights to access the setting panel') + @ns.expect(parser) + @ns.doc( + responses={ + 200: 'Success', + 400: 'Missing parameter', + 403: 'Insufficient permissions', + 500: 'Internal failure', + } + ) + def put(self, server=None): + """Creates a new client""" + newtemplate = self.parser.parse_args()['newtemplate'] + if not newtemplate: + self.abort(400, 'No template name provided') + parser = bui.client.get_parser(agent=server) + templates = parser.list_templates() + for tpl in templates: + if tpl['name'] == newtemplate: + self.abort(409, "Template '{}' already exists".format(newtemplate)) + # clientconfdir = bui.client.get_parser_attr('clientconfdir', server) + # if not clientconfdir: + # flash('Could not proceed, no \'clientconfdir\' find', 'warning') + # return redirect(request.referrer) + noti = bui.client.store_conf_cli(ImmutableMultiDict(), newtemplate, None, True, server) + if server: + noti.append([NOTIF_INFO, _('Click here to edit \'%(template)s\' configuration', url=url_for('view.cli_settings', server=server, client=newtemplate, template=True), template=newtemplate)]) + else: + noti.append([NOTIF_INFO, _('Click here to edit \'%(template)s\' configuration', url=url_for('view.cli_settings', client=newtemplate, template=True), template=newtemplate)]) + # clear the cache when we add a new client + cache.clear() + return {'notif': noti}, 201 + + @ns.route('/config', '//config', endpoint='new_client', @@ -267,7 +406,7 @@ class NewClientSettings(Resource): parser.add_argument('newclient', required=True, help="No 'newclient' provided") @api.disabled_on_demo() - @api.acl_admin_required(message='Sorry, you don\'t have rights to access the setting panel') + @api.acl_admin_or_moderator_required(message='Sorry, you don\'t have rights to access the setting panel') @ns.expect(parser) @ns.doc( responses={ @@ -282,7 +421,8 @@ class NewClientSettings(Resource): newclient = self.parser.parse_args()['newclient'] if not newclient: self.abort(400, 'No client name provided') - clients = bui.client.clients_list(server) + parser = bui.client.get_parser(agent=server) + clients = parser.list_clients() for cl in clients: if cl['name'] == newclient: self.abort(409, "Client '{}' already exists".format(newclient)) @@ -290,11 +430,11 @@ class NewClientSettings(Resource): # if not clientconfdir: # flash('Could not proceed, no \'clientconfdir\' find', 'warning') # return redirect(request.referrer) - noti = bui.client.store_conf_cli(ImmutableMultiDict(), newclient, None, server) + noti = bui.client.store_conf_cli(ImmutableMultiDict(), newclient, None, agent=server) if server: - noti.append([NOTIF_INFO, 'Click here to edit \'{}\' configuration'.format(url_for('view.cli_settings', server=server, client=newclient), newclient)]) + noti.append([NOTIF_INFO, _('Click here to edit \'%(client)s\' configuration', url=url_for('view.cli_settings', server=server, client=newclient), client=newclient)]) else: - noti.append([NOTIF_INFO, 'Click here to edit \'{}\' configuration'.format(url_for('view.cli_settings', client=newclient), newclient)]) + noti.append([NOTIF_INFO, _('Click here to edit \'%(client)s\' configuration', url=url_for('view.cli_settings', client=newclient), client=newclient)]) # clear the cache when we add a new client cache.clear() if bui.config['WITH_CELERY']: @@ -321,9 +461,15 @@ class ClientSettings(Resource): parser_delete.add_argument('revoke', type=boolean, help='Whether to revoke the certificate or not', default=False, nullable=True) parser_delete.add_argument('delcert', type=boolean, help='Whether to delete the certificate or not', default=False, nullable=True) parser_delete.add_argument('keepconf', type=boolean, help='Whether to keep the conf or not', default=False, nullable=True) + parser_delete.add_argument('template', type=boolean, help='Whether we work on a template or not', default=False, nullable=True) + parser_post = ns.parser() + parser_post.add_argument('template', type=boolean, help='Whether we work on a template or not', default=False, nullable=True) + parser_get = ns.parser() + parser_get.add_argument('template', type=boolean, help='Whether we work on a template or not', default=False, nullable=True) @api.disabled_on_demo() - @api.acl_admin_required(message='Sorry, you don\'t have rights to access the setting panel') + @api.acl_admin_or_moderator_required(message=_('Sorry, you don\'t have rights to access the setting panel')) + @ns.expect(parser_post) @ns.doc( responses={ 200: 'Success', @@ -333,10 +479,13 @@ class ClientSettings(Resource): ) def post(self, server=None, client=None, conf=None): """Saves a given client configuration""" - noti = bui.client.store_conf_cli(request.form, client, conf, server) + args = self.parser_post.parse_args() + template = args.get('template', False) + noti = bui.client.store_conf_cli(request.form, client, conf, template, server) return {'notif': noti} - @api.acl_admin_required(message='Sorry, you don\'t have rights to access the setting panel') + @api.acl_admin_or_moderator_required(message=_('Sorry, you don\'t have rights to access the setting panel')) + @ns.expect(parser_get) @ns.doc( responses={ 200: 'Success', @@ -350,29 +499,51 @@ class ClientSettings(Resource): conf = unquote(conf) except: pass - res = bui.client.read_conf_cli(client, conf, server) + args = self.parser_get.parse_args() + template = args.get('template', False) + parser = bui.client.get_parser(agent=server) + res = parser.read_client_conf(client, conf, template) refresh() # Translate the doc and placeholder API side - doc = bui.client.get_parser_attr('doc', server).copy() - placeholders = bui.client.get_parser_attr('placeholders', server).copy() - for key, val in iteritems(doc): - doc[key] = _(val) - for key, val in iteritems(placeholders): - placeholders[key] = _(val) + cache_keys = { + 'doc': '_doc_parser_{}-{}'.format(server, g.locale), + 'placeholders': '_placeholders_parser_{}-{}'.format(server, g.locale), + 'boolean_cli': '_boolean_cli_parser_{}'.format(server), + 'string_cli': '_string_cli_parser_{}'.format(server), + 'integer_cli': '_integer_cli_parser_{}'.format(server), + 'multi_cli': '_multi_cli_parser_{}'.format(server), + 'values': '_suggest_parser_{}'.format(server), + 'defaults': '_defaults_parser_{}'.format(server), + } + cache_results = {} + for name, key in iteritems(cache_keys): + if not cache.cache.has(key): + if name in ['doc', 'placeholders']: + _tmp = bui.client.get_parser_attr(name, server).copy() + _tmp2 = {} + for k, v in iteritems(_tmp): + _tmp2[k] = _(v) + cache_results[name] = _tmp2 + else: + cache_results[name] = bui.client.get_parser_attr(name, server) + cache.cache.set(key, cache_results[name], 3600) + else: + cache_results[name] = cache.cache.get(key) + return jsonify( results=res, - boolean=bui.client.get_parser_attr('boolean_cli', server), - string=bui.client.get_parser_attr('string_cli', server), - integer=bui.client.get_parser_attr('integer_cli', server), - multi=bui.client.get_parser_attr('multi_cli', server), - server_doc=doc, - suggest=bui.client.get_parser_attr('values', server), - placeholders=placeholders, - defaults=bui.client.get_parser_attr('defaults', server) + boolean=cache_results['boolean_cli'], + string=cache_results['string_cli'], + integer=cache_results['integer_cli'], + multi=cache_results['multi_cli'], + server_doc=cache_results['doc'], + suggest=cache_results['values'], + placeholders=cache_results['placeholders'], + defaults=cache_results['defaults'] ) @api.disabled_on_demo() - @api.acl_admin_required(message='Sorry, you don\'t have rights to access the setting panel') + @api.acl_admin_or_moderator_required(message=_('Sorry, you don\'t have rights to access the setting panel')) @ns.expect(parser_delete) @ns.doc( responses={ @@ -387,6 +558,7 @@ class ClientSettings(Resource): delcert = args.get('delcert', False) revoke = args.get('revoke', False) keepconf = args.get('keepconf', False) + template = args.get('template', False) if not keepconf: # clear the cache when we remove a client @@ -394,7 +566,8 @@ class ClientSettings(Resource): if bui.config['WITH_CELERY']: from ..tasks import force_scheduling_now force_scheduling_now() - return bui.client.delete_client(client, keepconf=keepconf, delcert=delcert, revoke=revoke, agent=server), 200 + parser = bui.client.get_parser(agent=server) + return parser.remove_client(client, keepconf, delcert, revoke, template), 200 @ns.route('/path-expander', @@ -414,7 +587,7 @@ class PathExpander(Resource): parser.add_argument('path', required=True, help="No 'path' provided") parser.add_argument('source', required=False, help="Which file is it included in") - @api.acl_admin_required(message='Sorry, you don\'t have rights to access the setting panel') + @api.acl_admin_or_moderator_required(message=_('Sorry, you don\'t have rights to access the setting panel')) @ns.doc( responses={ 200: 'Success', @@ -435,7 +608,8 @@ class PathExpander(Resource): path = unquote(path) if source: source = unquote(source) - paths = bui.client.expand_path(path, source, client, server) + parser = bui.client.get_parser(agent=server) + paths = parser.path_expander(path, source, client) if not paths: self.abort(403, 'Path not found') return {'result': paths} diff --git a/burpui/app.py b/burpui/app.py index 92e3616d..050b7319 100644 --- a/burpui/app.py +++ b/burpui/app.py @@ -10,349 +10,13 @@ jQuery/Bootstrap .. moduleauthor:: Ziirish """ import os -import re import logging -import warnings from logging import Formatter -from ._compat import PY3, to_unicode from .desc import __version__, __release__ - -if PY3: # pragma: no cover - basestring = str - - -def parse_db_setting(string): - parts = re.search( - '(?:(?P\w+)(?:\+(?P\w+))?://)?' - '(?:(?P\w+)(?::?(?P.+))?@)?' - '(?P[\w_.-]+):?(?P\d+)?(?:/(?P\w+))?', - string - ) - if not parts: # pragma: no cover - raise ValueError('Unable to parse the db: "{}"'.format(string)) - back = parts.group('backend') or '' - user = parts.group('user') or None - pwd = parts.group('pass') or None - host = parts.group('host') or '' - port = parts.group('port') or '' - db = parts.group('db') or '' - return (back, user, pwd, host, port, db) - - -def get_redis_server(myapp): - host = 'localhost' - port = 6379 - pwd = None - if myapp.redis and myapp.redis.lower() != 'none': - try: - back, user, pwd, host, port, db = parse_db_setting(myapp.redis) - host = host or 'localhost' - try: - port = int(port) - except (ValueError, IndexError): - port = 6379 - except ValueError: # pragma: no cover - pass - return host, port, pwd - - -def create_db(myapp, cli=False, unittest=False, create=True, celery_worker=False): - """Create the SQLAlchemy instance if possible - - :param myapp: Application context - :type myapp: :class:`burpui.server.BUIServer` - """ - if myapp.config['WITH_SQL']: - try: - from .ext.sql import db - from sqlalchemy.exc import OperationalError - from sqlalchemy_utils.functions import database_exists - myapp.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - if not database_exists(myapp.config['SQLALCHEMY_DATABASE_URI']) and \ - not cli and not unittest and not celery_worker: - if create: # pragma: no cover - import subprocess - local = os.path.join(os.getcwd(), '..', 'tools', 'bui-manage') - buimanage = local if os.path.exists(local) else 'bui-manage' - cmd = [ - buimanage, - '-c', - myapp.config['CFG'], - '-l', - os.devnull, - 'db', - 'upgrade' - ] - upgd = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - (out, _) = upgd.communicate() - if upgd.returncode != 0: - myapp.logger.error( - 'Disabling SQL support because ' - 'something went wrong while setting up the ' - 'database:\n{}'.format(out) - ) - myapp.config['WITH_SQL'] = False - return None - return create_db(myapp, cli, unittest, False) - else: # pragma: no cover - myapp.logger.error( - 'Database not found, disabling SQL support' - ) - myapp.config['WITH_SQL'] = False - return None - - back = parse_db_setting(myapp.config['SQLALCHEMY_DATABASE_URI'])[0] - - if 'mysql' in back: # pragma: no cover - # optimize SQL pools for MySQL driver - myapp.config['SQLALCHEMY_POOL_SIZE'] = 20 - myapp.config['SQLALCHEMY_POOL_RECYCLE'] = 600 - - db.init_app(myapp) - if not cli and not unittest and not celery_worker: # pragma: no cover - with myapp.app_context(): - try: - import subprocess - - # get the current revision from alembic_version - res = db.engine.execute( - 'select version_num from alembic_version' - ) - if not res: - raise Exception( - 'Alembic does not seem to be setup' - ) - current = None - for row in res: - current = to_unicode(row['version_num']) - break - - # get current head using alembic/FLask-Migrate - local = os.path.join( - os.getcwd(), - 'tools', - 'bui-manage' - ) - buimanage = local if os.path.exists(local) \ - else 'bui-manage' - cmd = [ - buimanage, - '-c', - myapp.config['CFG'], - '-l', - os.devnull, - 'db', - 'heads' - ] - rev = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - (out, _) = rev.communicate() - if rev.returncode != 0: - raise Exception( - 'something went wrong while setting up the ' - 'database:\n{}'.format(out) - ) - - latest = to_unicode(out).split()[0] - - # now we compare the revision numbers - if latest != current: - myapp.logger.critical( - 'Your database seems out of sync ({} != {}), ' - 'you may want to run \'bui-manage db ' - 'upgrade\'.'.format(latest, current) - ) - myapp.logger.critical( - 'Disabling SQL support for now.' - ) - myapp.config['WITH_SQL'] = False - return None - - except (OperationalError, Exception) as exp: - err = str(exp) - if 'no such table' in err: - myapp.logger.critical( - 'Your database seems out of sync, you may want ' - 'to run \'bui-manage db upgrade\'.' - ) - else: - myapp.logger.critical( - 'Something seems to be wrong with your setup: ' - '{}'.format(err) - ) - - myapp.logger.critical('Disabling SQL support for now.') - myapp.config['WITH_SQL'] = False - return None - - # If we are here, it means everything is alright - return db - - except ImportError: # pragma: no cover - myapp.logger.critical( - 'Unable to load requirements, you may want to run \'pip ' - 'install "burp-ui[sql]"\'.\nDisabling SQL support for now.' - ) - myapp.config['WITH_SQL'] = False - except OperationalError as exp: # pragma: no cover - myapp.logger.critical( - 'unable to contact database: {}\nDisabling SQL ' - 'support.'.format(exp) - ) - myapp.config['WITH_SQL'] = False - - return None - - -def create_websocket(myapp, websocket_server=False, celery_worker=False, - gunicorn=False, cli=False): - """Create the websocket server if possible - - :param myapp: Application context - :type myapp: :class:`burpui.server.BUIServer` - """ - if cli and not websocket_server: - return - broker = myapp.ws_broker - if broker is not False: - if not broker or broker is True: - broker = 'redis' - if broker and broker.lower() != 'none': - host, oport, pwd = get_redis_server(myapp) - odb = 4 - if broker.lower() not in ['default', 'redis']: - try: - (_, _, pwd, host, port, db) = parse_db_setting(myapp.use_celery) - if not port: - port = oport - if not db: - db = odb - else: - try: - db = int(db) - except ValueError: - db = odb - except ValueError: - pass - else: - port = oport - db = odb - if pwd: - redis_url = 'redis://:{}@{}:{}/{}'.format(pwd, host, port, db) - else: - redis_url = 'redis://{}:{}/{}'.format(host, port, db) - myapp.config['WS_MESSAGE_QUEUE'] = redis_url - myapp.config['WS_MANAGE_SESSION'] = not myapp.config.get('WITH_SRV_SESSION', False) - if os.getenv('BUI_MODE') == 'celery': - myapp.config['WS_ASYNC_MODE'] = 'threading' - # myapp.config['WS_ASYNC_MODE'] = 'threading' if not gunicorn else None - - if celery_worker: - return - - # if you are not a celery worker, we can patch the flask server - try: - from .ext.ws import socketio - socketio.init_app( - myapp, - message_queue=myapp.config.get('WS_MESSAGE_QUEUE'), - manage_session=myapp.config.get('WS_MANAGE_SESSION', False) - ) - myapp.config['WS_AVAILABLE'] = True - except ImportError: - myapp.config['WS_AVAILABLE'] = False - - # Now load the namespaces - if myapp.config['WITH_WS'] or websocket_server: - from .ws.namespace import BUINamespace - socketio.on_namespace(BUINamespace('/ws')) - - -def create_celery(myapp, warn=True): - """Create the Celery app if possible - - :param myapp: Application context - :type myapp: :class:`burpui.server.BUIServer` - """ - if myapp.config['WITH_CELERY']: # pragma: no cover - from .exceptions import BUIserverException - host, oport, pwd = get_redis_server(myapp) - odb = 2 - if isinstance(myapp.use_celery, basestring): - try: - (_, _, pwd, host, port, db) = parse_db_setting(myapp.use_celery) - if not port: - port = oport - if not db: - db = odb - else: - try: - db = int(db) - except ValueError: - db = odb - except ValueError: - pass - else: - db = odb - port = oport - if pwd: - redis_url = 'redis://:{}@{}:{}/{}'.format(pwd, host, port, db) - else: - redis_url = 'redis://{}:{}/{}'.format(host, port, db) - myapp.config['CELERY_BROKER_URL'] = myapp.config['BROKER_URL'] = \ - redis_url - myapp.config['CELERY_RESULT_BACKEND'] = redis_url - - from .ext.async import celery - celery.conf.update(myapp.config) - - if not hasattr(celery, 'flask_app'): - celery.flask_app = myapp - - TaskBase = celery.Task - - class ContextTask(TaskBase): - abstract = True - - def __call__(self, *args, **kwargs): - with myapp.app_context(): - try: - return TaskBase.__call__(self, *args, **kwargs) - except BUIserverException: - # ignore unhandled exceptions in the celery worker - pass - - celery.Task = ContextTask - - # may fail in case redis is not running (this can happen while running - # the bui-manage script) - try: - from .tasks import force_scheduling_now - force_scheduling_now() - except: # pragma: no cover - pass - - return celery - - if warn: # pragma: no cover - message = 'Something went wrong while initializing celery worker.\n' \ - 'Maybe it is not enabled in your conf ' \ - '({}).'.format(myapp.config['CFG']) - warnings.warn( - message, - RuntimeWarning - ) - - return None +from .extensions import create_celery, create_db, create_websocket, \ + parse_db_setting, get_redis_server def create_app(conf=None, verbose=0, logfile=None, **kwargs): @@ -380,7 +44,7 @@ def create_app(conf=None, verbose=0, logfile=None, **kwargs): :returns: A :class:`burpui.server.BUIServer` object """ - from flask import g, request, session + from flask import g, request, session, render_template from flask_login import LoginManager from flask_bower import Bower from flask_babel import gettext @@ -507,6 +171,15 @@ def create_app(conf=None, verbose=0, logfile=None, **kwargs): app.config['TEMPLATES_AUTO_RELOAD'] = True app.config['DEBUG'] = True + SENTRY_AVAILABLE = False + if app.demo: + try: + from .ext.sentry import sentry + sentry.init_app(app, dsn=app.config['BUI_DSN']) + SENTRY_AVAILABLE = True + except ImportError: + pass + # manage application secret key if app.secret_key and \ (app.secret_key.lower() == 'none' or @@ -739,6 +412,7 @@ def create_app(conf=None, verbose=0, logfile=None, **kwargs): @app.before_request def setup_request(): + g.version = '{}-{}'.format(__version__, __release__) g.locale = get_locale() g.date_format = session.get('dateFormat', 'llll') # make sure to store secure cookie if required @@ -749,6 +423,9 @@ def create_app(conf=None, verbose=0, logfile=None, **kwargs): ] app.config['SESSION_COOKIE_SECURE'] = \ app.config['REMEMBER_COOKIE_SECURE'] = any(criteria) + if '_extra' in request.args: + session['_extra'] = request.args.get('_extra') + g._extra = session.get('_extra', '') @app.login_manager.user_loader def load_user(userid): @@ -787,5 +464,15 @@ def create_app(conf=None, verbose=0, logfile=None, **kwargs): return app + if app.demo and SENTRY_AVAILABLE: + @app.errorhandler(500) + def internal_server_error(error): + from .ext.sentry import sentry + return render_template( + '500_sentry.html', + event_id=g.sentry_event_id, + public_dsn=sentry.client.get_public_dsn('https') + ) + init = create_app diff --git a/burpui/cli.py b/burpui/cli.py index 10eec87d..44f4f0c2 100644 --- a/burpui/cli.py +++ b/burpui/cli.py @@ -15,12 +15,13 @@ import click if os.getenv('BUI_MODE') in ['server', 'ws'] or 'websocket' in sys.argv: try: - import eventlet - eventlet.monkey_patch(socket=True) + from gevent import monkey + monkey.patch_socket() except ImportError: pass from .app import create_app # noqa +from .exceptions import BUIserverException # noqa from six import iteritems # noqa try: @@ -302,6 +303,7 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry): import difflib import tempfile + parser = app.client.get_parser() orig = source = None conf_orig = [] if dry: @@ -315,11 +317,26 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry): (_, temp) = tempfile.mkstemp() app.conf.options.filename = temp + # handle migration of old config files + if app.conf.section_exists('Burp2'): + if app.conf.rename_section('Burp2', 'Burp', source): + click.echo( + click.style( + 'Renaming old [Burp2] section', + fg='blue' + ) + ) + app.conf._refresh(True) + + refresh = False if not app.conf.lookup_section('Burp', source): - app.conf._refresh(True) + refresh = True if not app.conf.lookup_section('Global', source): - app.conf._refresh(True) + refresh = True if (database or redis) and not app.conf.lookup_section('Production', source): + refresh = True + + if refresh: app.conf._refresh(True) def _edit_conf(key, val, attr, section='Burp', obj=app.client): @@ -329,7 +346,18 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry): getattr(obj, attr) != val): app.conf.options[section][key] = val app.conf.options.write() - app.conf._refresh(True) + click.echo( + click.style( + 'Adding new option: "{}={}" to section [{}]'.format( + key, + val, + section + ), + fg='blue' + ) + ) + return True + return False def _color_diff(line): if line.startswith('+'): @@ -340,9 +368,13 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry): return click.style(line, fg='blue') return line - _edit_conf('bconfcli', bconfcli, 'burpconfcli') - _edit_conf('bconfsrv', bconfsrv, 'burpconfsrv') - _edit_conf('plugins', plugins, 'plugins', 'Global', app) + refresh = False + refresh |= _edit_conf('bconfcli', bconfcli, 'burpconfcli') + refresh |= _edit_conf('bconfsrv', bconfsrv, 'burpconfsrv') + refresh |= _edit_conf('plugins', plugins, 'plugins', 'Global', app) + + if refresh: + app.conf._refresh(True) if redis: try: @@ -446,87 +478,6 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry): getattr(app.client, 'burpconfsrv') dest_bconfcli = bconfcli - if not os.path.exists(bconfcli): - clitpl = """ -mode = client -port = 4971 -status_port = 4972 -server = ::1 -password = abcdefgh -cname = {0} -protocol = 1 -pidfile = /tmp/burp.client.pid -syslog = 0 -stdout = 1 -progress_counter = 1 -network_timeout = 72000 -server_can_restore = 0 -cross_all_filesystems=0 -ca_burp_ca = /usr/sbin/burp_ca -ca_csr_dir = /etc/burp/CA-client -ssl_cert_ca = /etc/burp/ssl_cert_ca-client-{0}.pem -ssl_cert = /etc/burp/ssl_cert-bui-client.pem -ssl_key = /etc/burp/ssl_cert-bui-client.key -ssl_key_password = password -ssl_peer_cn = burpserver -include = /home -exclude_fs = sysfs -exclude_fs = tmpfs -nobackup = .nobackup -exclude_comp=bz2 -exclude_comp=gz -""".format(client) - - if dry: - (_, dest_bconfcli) = tempfile.mkstemp() - with open(dest_bconfcli, 'w') as confcli: - confcli.write(clitpl) - - parser = app.client.get_parser() - - confcli = Config(dest_bconfcli, parser, 'srv') - confcli.set_default(dest_bconfcli) - confcli.parse() - - if confcli.get('cname') != client: - confcli['cname'] = client - if confcli.get('server') != host: - confcli['server'] = host - - if confcli.dirty: - if dry: - (_, dstfile) = tempfile.mkstemp() - else: - dstfile = bconfcli - - confcli.store(conf=bconfcli, dest=dstfile, insecure=True) - if dry: - before = [] - after = [] - try: - with open(bconfcli) as fil: - before = fil.readlines() - except: - pass - try: - with open(dstfile) as fil: - after = fil.readlines() - os.unlink(dstfile) - except: - pass - - if dest_bconfcli != bconfcli: - # the file did not exist - os.unlink(dest_bconfcli) - before = [] - - diff = difflib.unified_diff(before, after, fromfile=bconfcli, tofile='{}.new'.format(bconfcli)) - out = '' - for line in diff: - out += _color_diff(line) - if out: - click.echo_via_pager(out) - if not os.path.exists(bconfsrv): click.echo( click.style( @@ -560,8 +511,8 @@ exclude_comp=gz ) ) + status_port = confsrv.get('status_port', [4972]) if 'max_status_children' not in confsrv: - confsrv['max_status_children'] = 15 click.echo( click.style( 'We need to set the number of \'max_status_children\'. ' @@ -569,9 +520,20 @@ exclude_comp=gz fg='blue' ) ) + confsrv['max_status_children'] = 15 + status_port = status_port[0] else: max_status_children = confsrv.get('max_status_children') - if max_status_children < 15: + found = False + for idx, value in enumerate(max_status_children): + if value >= 15: + found = True + if idx >= len(status_port): + status_port = status_port[-1] + else: + status_port = status_port[idx] + break + if not found: click.echo( click.style( 'We need to raise the number of \'max_status_children\'. ' @@ -579,7 +541,8 @@ exclude_comp=gz fg='yellow' ) ) - confsrv['max_status_children'] = 15 + confsrv['max_status_children'][-1] = 15 + status_port = status_port[-1] if 'restore_client' not in confsrv: confsrv['restore_client'] = client @@ -643,6 +606,90 @@ exclude_comp=gz ) bconfagent = os.devnull + if not os.path.exists(bconfcli): + clitpl = """ +mode = client +port = 4971 +status_port = 4972 +server = ::1 +password = abcdefgh +cname = {0} +protocol = 1 +pidfile = /tmp/burp.client.pid +syslog = 0 +stdout = 1 +progress_counter = 1 +network_timeout = 72000 +server_can_restore = 0 +cross_all_filesystems=0 +ca_burp_ca = /usr/sbin/burp_ca +ca_csr_dir = /etc/burp/CA-client +ssl_cert_ca = /etc/burp/ssl_cert_ca-client-{0}.pem +ssl_cert = /etc/burp/ssl_cert-bui-client.pem +ssl_key = /etc/burp/ssl_cert-bui-client.key +ssl_key_password = password +ssl_peer_cn = burpserver +include = /home +exclude_fs = sysfs +exclude_fs = tmpfs +nobackup = .nobackup +exclude_comp=bz2 +exclude_comp=gz +""".format(client) + + if dry: + (_, dest_bconfcli) = tempfile.mkstemp() + with open(dest_bconfcli, 'w') as confcli: + confcli.write(clitpl) + + parser = app.client.get_parser() + + confcli = Config(dest_bconfcli, parser, 'srv') + confcli.set_default(dest_bconfcli) + confcli.parse() + + if confcli.get('cname') != client: + confcli['cname'] = client + if confcli.get('server') != host: + confcli['server'] = host + if confcli.get('status_port')[0] != status_port: + c_status_port = confcli.get_raw('status_port') + c_status_port[0] = status_port + + if confcli.dirty: + if dry: + (_, dstfile) = tempfile.mkstemp() + else: + dstfile = bconfcli + + confcli.store(conf=bconfcli, dest=dstfile, insecure=True) + if dry: + before = [] + after = [] + try: + with open(bconfcli) as fil: + before = fil.readlines() + except: + pass + try: + with open(dstfile) as fil: + after = fil.readlines() + os.unlink(dstfile) + except: + pass + + if dest_bconfcli != bconfcli: + # the file did not exist + os.unlink(dest_bconfcli) + before = [] + + diff = difflib.unified_diff(before, after, fromfile=bconfcli, tofile='{}.new'.format(bconfcli)) + out = '' + for line in diff: + out += _color_diff(line) + if out: + click.echo_via_pager(out) + if not os.path.exists(bconfagent): agenttpl = """ @@ -773,9 +820,28 @@ def diag(client, host, tips): ) ) - bconfcli = app.conf.options.get('Burp', {}).get('bconfcli') or \ + section = 'Burp' + if not app.conf.section_exists(section): + click.echo( + click.style( + 'Section [Burp] not found, looking for the old [Burp2] section ' + 'instead.', + fg='yellow' + ) + ) + section = 'Burp2' + if not app.conf.section_exists(section): + click.echo( + click.style( + 'No [Burp*] section found at all!', + fg='red' + ) + ) + section = 'Burp' + + bconfcli = app.conf.options.get(section, {}).get('bconfcli') or \ getattr(app.client, 'burpconfcli') - bconfsrv = app.conf.options.get('Burp', {}).get('bconfsrv') or \ + bconfsrv = app.conf.options.get(section, {}).get('bconfsrv') or \ getattr(app.client, 'burpconfsrv') try: @@ -840,6 +906,8 @@ def diag(client, host, tips): errors = True if os.path.exists(bconfsrv): + parser = app.client.get_parser() + confsrv = Config(bconfsrv, parser, 'srv') confsrv.set_default(bconfsrv) confsrv.parse() @@ -863,8 +931,8 @@ def diag(client, host, tips): ) ) - max_status_children = confsrv.get('max_status_children', -1) - if 'max_status_children' not in confsrv or max_status_children < 15: + max_status_children = confsrv.get('max_status_children', [-1]) + if all([x < 15 for x in max_status_children]): click.echo( click.style( '\'max_status_children\' is to low, you need to set it to ' @@ -995,16 +1063,69 @@ def diag(client, host, tips): @click.option('-v', '--verbose', is_flag=True, help='Dump parts of the config (Please double check no sensitive' ' data leaked)') -def sysinfo(verbose): +@click.option('-l', '--load', is_flag=True, + help='Load all configured modules for full summary') +def sysinfo(verbose, load): """Returns a couple of system informations to help debugging.""" from .desc import __release__, __version__ + import platform + + msg = None + if load: + try: + msg = app.load_modules(True) + except Exception as e: + msg = str(e) + + backend_version = app.vers + if not app.standalone: + backend_version = 'multi' + + colors = { + 'True': 'green', + 'False': 'red', + } + embedded_ws = str(app.websocket) + available_ws = str(WS_AVAILABLE) + click.echo('Python version: {}.{}.{}'.format(sys.version_info[0], sys.version_info[1], sys.version_info[2])) click.echo('Burp-UI version: {} ({})'.format(__version__, __release__)) + click.echo('OS: {}:{} ({})'.format(platform.system(), platform.release(), os.name)) + if platform.system() == 'Linux': + click.echo('Distribution: {} {} {}'.format(*platform.dist())) click.echo('Single mode: {}'.format(app.standalone)) - click.echo('Backend version: {}'.format(app.vers)) - click.echo('WebSocket embedded: {}'.format(app.websocket)) - click.echo('WebSocket available: {}'.format(WS_AVAILABLE)) + click.echo('Backend version: {}'.format(backend_version)) + click.echo('WebSocket embedded: {}'.format(click.style(embedded_ws, fg=colors[embedded_ws]))) + click.echo('WebSocket available: {}'.format(click.style(available_ws, colors[available_ws]))) click.echo('Config file: {}'.format(app.config.conffile)) + if load: + if not app.standalone and not msg: + click.echo('Agents:') + for agent, obj in iteritems(app.client.servers): + client_version = server_version = 'unknown' + try: + app.client.status(agent=agent) + client_version = app.client.get_client_version(agent=agent) + server_version = app.client.get_server_version(agent=agent) + except BUIserverException: + pass + alive = obj.ping() + if alive: + status = click.style('ALIVE', fg='green') + else: + status = click.style('DISCONNECTED', fg='red') + click.echo(' - {} ({})'.format(agent, status)) + click.echo(' * client version: {}'.format(client_version)) + click.echo(' * server version: {}'.format(server_version)) + elif not msg: + server_version = 'unknown' + try: + app.client.status() + server_version = app.client.get_server_version() + except BUIserverException: + pass + click.echo('Burp client version: {}'.format(app.client.client_version)) + click.echo('Burp server version: {}'.format(server_version)) if verbose: click.echo('>>>>> Extra verbose informations:') click.echo(click.style( @@ -1025,3 +1146,6 @@ def sysinfo(verbose): for key, val in iteritems(app.config.options.get(section, {})): click.echo(' {} = {}'.format(key, val)) click.echo(' 8<{}END[{}]'.format('-' * (69 - len(section)), section)) + + if load and msg: + _die(msg, 'sysinfo') diff --git a/burpui/config.py b/burpui/config.py index 84b1b9e7..84356fb8 100644 --- a/burpui/config.py +++ b/burpui/config.py @@ -130,7 +130,7 @@ class BUIConfig(dict): conf. """ ret = True - if section not in self.options: + if not self.section_exists(section): # look for the section in the comments conffile = self.options.filename source = source or conffile @@ -154,6 +154,30 @@ class BUIConfig(dict): ret = False return ret + def section_exists(self, section): + """Check whether a section exists or not""" + return section in self.options + + def rename_section(self, old_section, new_section, source=None): + """Rename a given section""" + ret = False + if not self.section_exists(old_section): + return ret + conffile = self.options.filename + source = source or conffile + ori = [] + with codecs.open(source, 'r', 'utf-8', errors='ignore') as config: + ori = [x.rstrip('\n') for x in config.readlines()] + if ori: + with codecs.open(conffile, 'w', 'utf-8', errors='ignore') as config: + for line in ori: + if re.match(r'^\s*(#|;)+\s*\[{}\]'.format(old_section), line): + config.write('{}\n'.format(line.replace(old_section, new_section))) + ret = True + else: + config.write('{}\n'.format(line)) + return ret + def changed(self, id): """Check if the conf has changed""" if (datetime.datetime.now() - self.last) > self.delta: @@ -299,6 +323,8 @@ class BUIConfig(dict): "'{}': no such validator".format(cast) ) return val + if cast == 'force_list' and val is None: + val = [] ret = caster(val) # special case for boolean and integer, etc. if ret is None: @@ -314,7 +340,7 @@ class BUIConfig(dict): ) except validate.ValidateError as exp: ret = default - self.logger.warning( + self.logger.info( '{}\n[{}]:{} - found: {}, default: {} -> {}'.format( str(exp), section, diff --git a/burpui/ext/sentry.py b/burpui/ext/sentry.py new file mode 100644 index 00000000..cec8de14 --- /dev/null +++ b/burpui/ext/sentry.py @@ -0,0 +1,12 @@ +# -*- coding: utf8 -*- +""" +.. module:: burpui.ext.sentry + :platform: Unix + :synopsis: Burp-UI external Sentry module. + +.. moduleauthor:: Ziirish + +""" +from raven.contrib.flask import Sentry + +sentry = Sentry() diff --git a/burpui/ext/ws.py b/burpui/ext/ws.py index eaf166a6..e28035ef 100644 --- a/burpui/ext/ws.py +++ b/burpui/ext/ws.py @@ -7,32 +7,17 @@ .. moduleauthor:: Ziirish """ -import os -import sys - from ..config import config from flask_socketio import SocketIO -_stdout = sys.stdout -_stderr = sys.stderr -null = open(os.devnull, 'wb') - -# hide stdout and stderr messages -sys.stdout = sys.stderr = null - options = {} if config.get('WS_ASYNC_MODE'): options['async_mode'] = config.get('WS_ASYNC_MODE') +# engineio_logger=config.get('WS_DEBUG', False), socketio = SocketIO( message_queue=config.get('WS_MESSAGE_QUEUE'), manage_session=config.get('WS_MANAGE_SESSION', False), - engineio_logger=config.get('WS_DEBUG', False), + engineio_logger=False, **options ) - -# revert stdout and stderr -sys.stdout = _stdout -sys.stderr = _stderr -# null.flush() -# null.close() diff --git a/burpui/extensions.py b/burpui/extensions.py new file mode 100644 index 00000000..f82ff997 --- /dev/null +++ b/burpui/extensions.py @@ -0,0 +1,349 @@ +# -*- coding: utf8 -*- +""" + +.. module:: burpui.extensions + :platform: Unix + :synopsis: Burp-UI extensions module. + +.. moduleauthor:: Ziirish +""" +import os +import re +import warnings + +from ._compat import PY3, to_unicode + +if PY3: # pragma: no cover + basestring = str + + +def parse_db_setting(string): + parts = re.search( + '(?:(?P\w+)(?:\+(?P\w+))?://)?' + '(?:(?P\w+)(?::?(?P.+))?@)?' + '(?P[\w_.-]+):?(?P\d+)?(?:/(?P\w+))?', + string + ) + if not parts: # pragma: no cover + raise ValueError('Unable to parse the db: "{}"'.format(string)) + back = parts.group('backend') or '' + user = parts.group('user') or None + pwd = parts.group('pass') or None + host = parts.group('host') or '' + port = parts.group('port') or '' + db = parts.group('db') or '' + return (back, user, pwd, host, port, db) + + +def get_redis_server(myapp): + host = 'localhost' + port = 6379 + pwd = None + if myapp.redis and myapp.redis.lower() != 'none': + try: + back, user, pwd, host, port, db = parse_db_setting(myapp.redis) + host = host or 'localhost' + try: + port = int(port) + except (ValueError, IndexError): + port = 6379 + except ValueError: # pragma: no cover + pass + return host, port, pwd + + +def create_db(myapp, cli=False, unittest=False, create=True, celery_worker=False): + """Create the SQLAlchemy instance if possible + + :param myapp: Application context + :type myapp: :class:`burpui.server.BUIServer` + """ + if myapp.config['WITH_SQL']: + try: + from .ext.sql import db + from sqlalchemy.exc import OperationalError + from sqlalchemy_utils.functions import database_exists + myapp.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False + if not database_exists(myapp.config['SQLALCHEMY_DATABASE_URI']) and \ + not cli and not unittest and not celery_worker: + if create: # pragma: no cover + import subprocess + local = os.path.join(os.getcwd(), '..', 'tools', 'bui-manage') + buimanage = local if os.path.exists(local) else 'bui-manage' + cmd = [ + buimanage, + '-c', + myapp.config['CFG'], + '-l', + os.devnull, + 'db', + 'upgrade' + ] + upgd = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + (out, _) = upgd.communicate() + if upgd.returncode != 0: + myapp.logger.error( + 'Disabling SQL support because ' + 'something went wrong while setting up the ' + 'database:\n{}'.format(out) + ) + myapp.config['WITH_SQL'] = False + return None + return create_db(myapp, cli, unittest, False) + else: # pragma: no cover + myapp.logger.error( + 'Database not found, disabling SQL support' + ) + myapp.config['WITH_SQL'] = False + return None + + back = parse_db_setting(myapp.config['SQLALCHEMY_DATABASE_URI'])[0] + + if 'mysql' in back: # pragma: no cover + # optimize SQL pools for MySQL driver + myapp.config['SQLALCHEMY_POOL_SIZE'] = 20 + myapp.config['SQLALCHEMY_POOL_RECYCLE'] = 600 + + db.init_app(myapp) + if not cli and not unittest and not celery_worker: # pragma: no cover + with myapp.app_context(): + try: + import subprocess + + # get the current revision from alembic_version + res = db.engine.execute( + 'select version_num from alembic_version' + ) + if not res: + raise Exception( + 'Alembic does not seem to be setup' + ) + current = None + for row in res: + current = to_unicode(row['version_num']) + break + + # get current head using alembic/FLask-Migrate + local = os.path.join( + os.getcwd(), + 'tools', + 'bui-manage' + ) + buimanage = local if os.path.exists(local) \ + else 'bui-manage' + cmd = [ + buimanage, + '-c', + myapp.config['CFG'], + '-l', + os.devnull, + 'db', + 'heads' + ] + rev = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + (out, _) = rev.communicate() + if rev.returncode != 0: + raise Exception( + 'something went wrong while setting up the ' + 'database:\n{}'.format(out) + ) + + latest = to_unicode(out).split()[0] + + # now we compare the revision numbers + if latest != current: + myapp.logger.critical( + 'Your database seems out of sync ({} != {}), ' + 'you may want to run \'bui-manage db ' + 'upgrade\'.'.format(latest, current) + ) + myapp.logger.critical( + 'Disabling SQL support for now.' + ) + myapp.config['WITH_SQL'] = False + return None + + except (OperationalError, Exception) as exp: + err = str(exp) + if 'no such table' in err: + myapp.logger.critical( + 'Your database seems out of sync, you may want ' + 'to run \'bui-manage db upgrade\'.' + ) + else: + myapp.logger.critical( + 'Something seems to be wrong with your setup: ' + '{}'.format(err) + ) + + myapp.logger.critical('Disabling SQL support for now.') + myapp.config['WITH_SQL'] = False + return None + + # If we are here, it means everything is alright + return db + + except ImportError: # pragma: no cover + myapp.logger.critical( + 'Unable to load requirements, you may want to run \'pip ' + 'install "burp-ui[sql]"\'.\nDisabling SQL support for now.' + ) + myapp.config['WITH_SQL'] = False + except OperationalError as exp: # pragma: no cover + myapp.logger.critical( + 'unable to contact database: {}\nDisabling SQL ' + 'support.'.format(exp) + ) + myapp.config['WITH_SQL'] = False + + return None + + +def create_websocket(myapp, websocket_server=False, celery_worker=False, + gunicorn=False, cli=False): + """Create the websocket server if possible + + :param myapp: Application context + :type myapp: :class:`burpui.server.BUIServer` + """ + if cli and not websocket_server: + return + broker = myapp.ws_broker + if broker is not False: + if not broker or broker is True: + broker = 'redis' + if broker and broker.lower() != 'none': + host, oport, pwd = get_redis_server(myapp) + odb = 4 + if broker.lower() not in ['default', 'redis']: + try: + (_, _, pwd, host, port, db) = parse_db_setting(myapp.use_celery) + if not port: + port = oport + if not db: + db = odb + else: + try: + db = int(db) + except ValueError: + db = odb + except ValueError: + pass + else: + port = oport + db = odb + if pwd: + redis_url = 'redis://:{}@{}:{}/{}'.format(pwd, host, port, db) + else: + redis_url = 'redis://{}:{}/{}'.format(host, port, db) + myapp.config['WS_MESSAGE_QUEUE'] = redis_url + myapp.config['WS_MANAGE_SESSION'] = not myapp.config.get('WITH_SRV_SESSION', False) + if os.getenv('BUI_MODE') == 'celery': + myapp.config['WS_ASYNC_MODE'] = 'threading' + # myapp.config['WS_ASYNC_MODE'] = 'threading' if not gunicorn else None + + if celery_worker: + return + + # if you are not a celery worker, we can patch the flask server + try: + from .ext.ws import socketio + socketio.init_app( + myapp, + message_queue=myapp.config.get('WS_MESSAGE_QUEUE'), + manage_session=myapp.config.get('WS_MANAGE_SESSION', False) + ) + myapp.config['WS_AVAILABLE'] = True + except ImportError: + myapp.config['WS_AVAILABLE'] = False + + # Now load the namespaces + if myapp.config['WITH_WS'] or websocket_server: + from .ws.namespace import BUINamespace + socketio.on_namespace(BUINamespace('/ws')) + + +def create_celery(myapp, warn=True): + """Create the Celery app if possible + + :param myapp: Application context + :type myapp: :class:`burpui.server.BUIServer` + """ + if myapp.config['WITH_CELERY']: # pragma: no cover + from .exceptions import BUIserverException + host, oport, pwd = get_redis_server(myapp) + odb = 2 + if isinstance(myapp.use_celery, basestring): + try: + (_, _, pwd, host, port, db) = parse_db_setting(myapp.use_celery) + if not port: + port = oport + if not db: + db = odb + else: + try: + db = int(db) + except ValueError: + db = odb + except ValueError: + pass + else: + db = odb + port = oport + if pwd: + redis_url = 'redis://:{}@{}:{}/{}'.format(pwd, host, port, db) + else: + redis_url = 'redis://{}:{}/{}'.format(host, port, db) + myapp.config['CELERY_BROKER_URL'] = myapp.config['BROKER_URL'] = \ + redis_url + myapp.config['CELERY_RESULT_BACKEND'] = redis_url + + from .ext.async import celery + celery.conf.update(myapp.config) + + if not hasattr(celery, 'flask_app'): + celery.flask_app = myapp + + TaskBase = celery.Task + + class ContextTask(TaskBase): + abstract = True + + def __call__(self, *args, **kwargs): + with myapp.app_context(): + try: + return TaskBase.__call__(self, *args, **kwargs) + except BUIserverException: + # ignore unhandled exceptions in the celery worker + pass + + celery.Task = ContextTask + + # may fail in case redis is not running (this can happen while running + # the bui-manage script) + try: + from .tasks import force_scheduling_now + force_scheduling_now() + except: # pragma: no cover + pass + + return celery + + if warn: # pragma: no cover + message = 'Something went wrong while initializing celery worker.\n' \ + 'Maybe it is not enabled in your conf ' \ + '({}).'.format(myapp.config['CFG']) + warnings.warn( + message, + RuntimeWarning + ) + + return None diff --git a/burpui/misc/acl/basic.py b/burpui/misc/acl/basic.py index 3d71a8a4..f6366fdd 100644 --- a/burpui/misc/acl/basic.py +++ b/burpui/misc/acl/basic.py @@ -1,10 +1,7 @@ # -*- coding: utf8 -*- -from .interface import BUIacl, BUIaclLoader -from ...utils import make_list - -import re -import json -import fnmatch +from .meta import meta_grants +from .interface import BUIaclLoader +from ...utils import NOTIF_OK, NOTIF_WARN, NOTIF_ERROR class ACLloader(BUIaclLoader): @@ -19,18 +16,22 @@ class ACLloader(BUIaclLoader): :type app: :class:`burpui.server.BUIServer` """ self.app = app + self.conf = self.app.conf self.admins = [ 'admin' ] self.moderators = [] - self.grants = {} - self.standalone = self.app.standalone - self.extended = False - self.legacy = False + self._groups = {} + self._grants = {} + self.first_setup = True self.moderator = {} - self._acl = None + self._acl = meta_grants self.conf_id = None - self.conf = self.app.conf + self.meta_id = meta_grants.id + meta_grants.register_backend(self.name, self) + self.load_acl(True) + + def reload(self): self.load_acl(True) def load_acl(self, force=False): @@ -38,11 +39,20 @@ class ACLloader(BUIaclLoader): if not self.conf.changed(self.conf_id): return False + # our config changed or we were forced to reload our rules. + # if the meta_grants didn't change, we reset them + # if they changed, it means something else triggered a reset + if not meta_grants.changed(self.meta_id) and not self.first_setup: + meta_grants.reset(self.name) + + self.first_setup = False + self.admins = [ 'admin' ] self.moderators = [] - self.grants = {} + self._grants = {} + self.groups_def = {} adms = [] mods = [] @@ -51,67 +61,285 @@ class ACLloader(BUIaclLoader): return not data or data == [None] if self.section in self.conf.options: - adms = self.conf.safe_get( - 'admin', - 'force_list', - section=self.section - ) - mods = self.conf.safe_get( - 'moderators', - 'force_list', - section=self.section - ) self.priority = self.conf.safe_get( 'priority', 'integer', section=self.section, defaults={self.section: {'priority': self.priority}} ) - self.extended = self.conf.safe_get( - 'extended', - 'boolean', + adms = self.conf.safe_get( + 'admin', + 'force_list', section=self.section ) - self.legacy = self.conf.safe_get( - 'legacy', - 'boolean', + mods = self.conf.safe_get( + '+moderator', + 'force_list', section=self.section ) self.moderator = self.conf.safe_get( - 'moderator', + '@moderator', 'force_string', section=self.section ) or {} + meta_grants.set_moderator_grants(self.moderator) for opt in self.conf.options.get(self.section).keys(): - if opt in ['admin', 'moderators', 'extended', 'priority', 'moderator', 'legacy']: + if opt in ['admin', '+moderator', 'priority', '@moderator']: continue record = self.conf.safe_get( opt, 'force_string', section=self.section ) + self.logger.debug('record: {} -> {}'.format(opt, record)) - self.grants[opt] = record + + def _record(key): + if gname not in self.groups_def: + self.groups_def[gname] = {} + self.groups_def[gname][key] = parsed + + return parsed + + if opt[0] == '+': + short = opt.lstrip('+') + gname = '@{}'.format(short) + parsed = meta_grants.set_group(gname, record) + self._groups[short] = parsed + _record('members') + elif opt[0] == '@': + short = opt.lstrip('@') + if short not in self._groups: + self._groups[short] = [] + gname = opt + parsed = record + _record('grants') + meta_grants.set_grant(gname, parsed) + else: + self._grants[opt] = record + meta_grants.set_grant(opt, record) if not is_empty(adms): self.admins = adms if not is_empty(mods): self.moderators = mods - if self.legacy: - self.extended = False + meta_grants.set_admin(self.admins) + meta_grants.set_moderator(self.moderators) self.logger.debug('admins: {}'.format(self.admins)) self.logger.debug('moderators: {}'.format(self.moderators)) self.logger.debug('moderator grants: {}'.format(self.moderator)) - self.logger.debug('extended: {}'.format(self.extended)) - self.logger.debug('legacy: {}'.format(self.legacy)) + self.logger.debug('groups: {}'.format(self.groups_def)) self.conf_id = self.conf.id - self._acl = BasicACL(self) + self.meta_id = meta_grants.id return True + def _setup_acl(self): + """Setup ACL management""" + if not self.conf.lookup_section(self.section): + self.conf._refresh() + + def add_grant(self, name, content): + """Add a grant""" + if name[0] in ['+', '@']: + message = "'{}' is not a valid grant name".format(name) + self.logger.error(message) + return False, message, NOTIF_ERROR + self._setup_acl() + if name in self.conf.options[self.section]: + message = "grant '{}' already exists".format(name) + self.logger.warning(message) + return False, message, NOTIF_WARN + self.conf.options[self.section][name] = content + self.conf.options.write() + self.load_acl(True) + message = "grant '{}' successfully added".format(name) + return True, message, NOTIF_OK + + def del_grant(self, name): + """Delete a grant""" + if name[0] in ['+', '@']: + message = "'{}' is not a valid grant name".format(name) + self.logger.error(message) + return False, message, NOTIF_ERROR + self._setup_acl() + self.load_acl(True) + if name not in self.conf.options[self.section]: + message = "grant '{}' does not exist".format(name) + self.logger.error(message) + return False, message, NOTIF_ERROR + del self.conf.options[self.section][name] + self.conf.options.write() + self.load_acl(True) + message = "grant '{}' successfully removed".format(name) + return True, message, NOTIF_OK + + def mod_grant(self, name, content): + """Update a grant""" + if name[0] in ['+', '@']: + message = "'{}' is not a valid grant name".format(name) + self.logger.error(message) + return False, message, NOTIF_ERROR + self._setup_acl() + self.load_acl(True) + if name not in self.conf.options[self.section]: + message = "grant '{}' does not exist".format(name) + self.logger.error(message) + return False, message, NOTIF_WARN + self.conf.options[self.section][name] = content + self.conf.options.write() + self.load_acl(True) + message = "grant '{}' successfully modified".format(name) + return True, message, NOTIF_OK + + def add_group(self, name, content): + """Create a group""" + self._setup_acl() + name = '@{}'.format(name) + if name in self.conf.options[self.section]: + message = "group '{}' already exists".format(name) + self.logger.warning(message) + return False, message, NOTIF_WARN + self.conf.options[self.section][name] = content + self.conf.options.write() + self.load_acl(True) + message = "group '{}' successfully added".format(name) + return True, message, NOTIF_OK + + def del_group(self, name): + """Delete a group""" + self._setup_acl() + self.load_acl(True) + gname = '@{}'.format(name) + if gname not in self.conf.options[self.section]: + message = "group '{}' does not exist".format(name) + self.logger.error(message) + return False, message, NOTIF_ERROR + del self.conf.options[self.section][gname] + gmembers = '+{}'.format(name) + if gmembers in self.conf.options[self.section]: + del self.conf.options[self.section][gmembers] + self.conf.options.write() + self.load_acl(True) + message = "grant '{}' successfully removed".format(name) + return True, message, NOTIF_OK + + def mod_group(self, name, content): + """Update a group""" + self._setup_acl() + name = '@{}'.format(name) + if name not in self.conf.options[self.section]: + message = "group '{}' does not exist".format(name) + self.logger.warning(message) + return False, message, NOTIF_WARN + self.conf.options[self.section][name] = content + self.conf.options.write() + self.load_acl(True) + message = "group '{}' successfully modified".format(name) + return True, message, NOTIF_OK + + def add_group_member(self, group, member): + """Add a user to a group""" + if group not in self._groups: + message = "group '{}' does not exist".format(group) + self.logger.warning(message) + return False, message, NOTIF_WARN + if member in self._groups[group]: + message = "'{}' already in group '{}'".format(member, group) + self.logger.warning(message) + return False, message, NOTIF_WARN + self._setup_acl() + self._groups[group].append(member) + gmembers = '+{}'.format(group) + self.conf.options[self.section][gmembers] = ','.join(self._groups[group]) + self.conf.options.write() + self.load_acl(True) + message = "'{}' added to group '{}'".format(member, group) + return True, message, NOTIF_OK + + def del_group_member(self, group, member): + """Remove a user from a group""" + if group not in self._groups: + message = "group '{}' does not exist".format(group) + self.logger.warning(message) + return False, message, NOTIF_WARN + if member not in self._groups[group]: + message = "'{}' not in group '{}'".format(member, group) + self.logger.warning(message) + return False, message, NOTIF_WARN + self._setup_acl() + self._groups[group].remove(member) + gmembers = '+{}'.format(group) + self.conf.options[self.section][gmembers] = ','.join(self._groups[group]) + self.conf.options.write() + self.load_acl(True) + message = "'{}' removed from group '{}'".format(member, group) + return True, message, NOTIF_OK + + def add_moderator(self, member): + """Add a moderator""" + if member in self.moderators: + message = "'{}' is already a moderator".format(member) + self.logger.warning(message) + return False, message, NOTIF_WARN + self._setup_acl() + self.moderators.append(member) + self.conf.options[self.section]['+moderator'] = ','.format(self.moderators) + self.conf.options.write() + message = "'{}' successfully added as moderator".format(member) + return True, message, NOTIF_OK + + def del_moderator(self, member): + """Delete a moderator""" + if member not in self.moderators: + message = "'{}' is not a moderator".format(member) + self.logger.warning(message) + return False, message, NOTIF_WARN + self._setup_acl() + self.moderators.remove(member) + self.conf.options[self.section]['+moderator'] = ','.format(self.moderators) + self.conf.options.write() + message = "'{}' successfully removed from moderators".format(member) + return True, message, NOTIF_OK + + def mod_moderator(self, grants): + """Update moderator grants""" + self._setup_acl() + self.moderator = grants + self.conf.options[self.section]['@moderator'] = grants + self.conf.options.write() + message = "moderator grants updated" + return True, message, NOTIF_OK + + def add_admin(self, member): + """Add an admin""" + if member in self.admins: + message = "'{}' is already an admin".format(member) + self.logger.warning(message) + return False, message, NOTIF_WARN + self._setup_acl() + self.admins.append(member) + self.conf.options[self.section]['admin'] = ','.format(self.admins) + self.conf.options.write() + message = "'{}' successfully added as admin".format(member) + return True, message, NOTIF_OK + + def del_admin(self, member): + """Delete an admin""" + if member in self.admins: + message = "'{}' is not an admin".format(member) + self.logger.warning(message) + return False, message, NOTIF_WARN + self._setup_acl() + self.admins.remove(member) + self.conf.options[self.section]['admin'] = ','.format(self.admins) + self.conf.options.write() + message = "'{}' successfully removed from admins".format(member) + return True, message, NOTIF_OK + @property def acl(self): """Property to retrieve the backend""" @@ -120,168 +348,12 @@ class ACLloader(BUIaclLoader): return self._acl return None # pragma: no cover + @property + def grants(self): + """Property to retrieve the list of grants""" + return self._grants -class BasicACL(BUIacl): - """See :class:`burpui.misc.acl.interface.BUIacl`""" - def __init__(self, loader=None): - """:func:`burpui.misc.acl.interface.BUIacl.__init__` instanciate ACL - engine. - - :param loader: ACL loader - :type loader: :class:`burpui.misc.acl.interface.BUIaclLoader` - """ - if not loader: # pragma: no cover - return - self.loader = loader - self.standalone = loader.standalone - self.admins = loader.admins - self.moderators = loader.moderators - self.moderator = loader.moderator - self.grants = loader.grants - self.extended = loader.extended - self.legacy = loader.legacy - self._parsed_grants = [] - self._clients_cache = {} - self._agents_cache = {} - self._advanced_cache = {} - - def is_admin(self, username=None): - """See :func:`burpui.misc.acl.interface.BUIacl.is_admin`""" - if not username: # pragma: no cover - return False - return username in self.admins - - def is_moderator(self, username=None): - """See :func:`burpui.misc.acl.interface.BUIacl.is_moderator`""" - if not username: - return False - return username in self.moderators - - def _extract_grants(self, username): - if username not in self._parsed_grants: - - if username == 'moderator': - grants = self.moderator - else: - grants = self.grants.get(username, '') - try: - grants = json.loads(grants) - except: - if ',' in grants: - grants = grants.split(',') - else: - grants = make_list(grants) - - clients, agents, advanced = self._parse_clients(grants) - self._clients_cache[username] = clients - self._agents_cache[username] = agents - self._advanced_cache[username] = advanced - - if self.is_moderator(username): - if 'moderator' not in self._parsed_grants: - self._extract_grants('moderator') - self._clients_cache[username] = self._merge_data( - self._clients_cache[username], - self._clients_cache.get('moderator', []) - ) - self._agents_cache[username] = self._merge_data( - self._agents_cache[username], - self._agents_cache.get('moderator', []) - ) - self._advanced_cache[username] = self._merge_data( - self._advanced_cache[username], - self._advanced_cache.get('moderator', []) - ) - - self._parsed_grants.append(username) - - def _extract_clients(self, username): - if username not in self._parsed_grants: - self._extract_grants(username) - return self._clients_cache.get(username, []) - - def _extract_agents(self, username): - if username not in self._parsed_grants: - self._extract_grants(username) - return self._agents_cache.get(username, []) - - def _extract_advanced(self, username): - if username not in self._parsed_grants: - self._extract_grants(username) - return self._advanced_cache.get(username, {}) - - def _extract_advanced_mode(self, username, mode, kind): - if username not in self._parsed_grants: - self._extract_grants(username) - return self._advanced_cache.get(username, {}).get(mode, {}).get(kind, []) - - def _client_match(self, username, client): - clients = self._extract_clients(username) - if not clients: - return None - - if self.extended: - for exp in clients: - regex = fnmatch.translate(exp) - if re.match(regex, client): - return exp - return False - else: - return client if client in clients else False - - def _server_match(self, username, server): - servers = self._extract_agents(username) - if not servers: - return None - - if self.extended: - for exp in servers: - regex = fnmatch.translate(exp) - if re.match(regex, server): - return exp - return False - else: - return server if server in servers else False - - def is_client_allowed(self, username=None, client=None, server=None): - """See :func:`burpui.misc.acl.interface.BUIacl.is_client_allowed`""" - if not username or not client: # pragma: no cover - return False - - is_admin = self.is_admin(username) - client_match = self._client_match(username, client) - - if not client_match and username == client: - client_match = username - - if server: - server_match = self._server_match(username, server) - if server_match is not None or self.legacy: - if not server_match: - return is_admin - - advanced = self._extract_advanced(username) - - if not client_match and server_match not in advanced and \ - (server_match in self._extract_advanced_mode(username, 'ro', 'agents') or - server_match in self._extract_advanced_mode(username, 'rw', 'agents')): - return True - - advanced = advanced.get(server_match, advanced.get(server, [])) - if client_match not in advanced and client not in advanced: - return is_admin - - return client_match is not False or is_admin - - def is_server_allowed(self, username=None, server=None): - """See :func:`burpui.misc.acl.interface.BUIacl.is_server_allowed`""" - if not username or not server: - return False - - server_match = self._server_match(username, server) - is_admin = self.is_admin(username) - - if server_match is None and self.legacy: - server_match = False - - return server_match is not False or is_admin + @property + def groups(self): + """Property to retrieve the list of groups with their members""" + return self.groups_def diff --git a/burpui/misc/acl/handler.py b/burpui/misc/acl/handler.py index 184d3da3..974c2d93 100644 --- a/burpui/misc/acl/handler.py +++ b/burpui/misc/acl/handler.py @@ -2,6 +2,7 @@ import os from .interface import BUIacl, BUIaclLoader +from .meta import meta_grants from importlib import import_module from six import iteritems @@ -9,6 +10,8 @@ from collections import OrderedDict class ACLloader(BUIaclLoader): + section = name = 'ACL' + def __init__(self, app=None): """See :func:`burpui.misc.acl.interface.BUIaclLoader.__init__` @@ -16,9 +19,29 @@ class ACLloader(BUIaclLoader): :type app: :class:`burpui.server.BUIServer` """ self.app = app + self.conf = self.app.conf self._acl = ACLhandler(self) backends = [] self.errors = {} + if self.section in self.conf.options: + opts = {} + opts['extended'] = self.conf.safe_get( + 'extended', + 'boolean', + section=self.section + ) + opts['assume_granted'] = self.conf.safe_get( + 'assume_granted', + 'boolean', + section=self.section, + defaults={self.section: {'assume_granted': True}} + ) + opts['legacy'] = self.conf.safe_get( + 'legacy', + 'boolean', + section=self.section + ) + meta_grants.options = opts if self.app.acl_engine and 'none' not in self.app.acl_engine: me, _ = os.path.splitext(os.path.basename(__file__)) back = self.app.acl_engine @@ -41,10 +64,10 @@ class ACLloader(BUIaclLoader): except: import traceback self.errors[name] = traceback.format_exc() - backends.sort(key=lambda x: x.priority, reverse=True) + backends.sort(key=lambda x: getattr(x, 'priority', -1), reverse=True) if not backends: raise ImportError( - 'No backend found for \'{}\':\n{}'.format(self.app.auth, + 'No backend found for \'{}\':\n{}'.format(self.app.acl_engine, self.errors) ) for name, err in iteritems(self.errors): @@ -55,10 +78,21 @@ class ACLloader(BUIaclLoader): for obj in backends: self.backends[obj.name] = obj + def reload(self): + return None + @property def acl(self): return self._acl + @property + def grants(self): + return meta_grants.grants + + @property + def groups(self): + return meta_grants.groups + class ACLhandler(BUIacl): """See :class:`burpui.misc.acl.interface.BUIacl`""" @@ -78,6 +112,9 @@ class ACLhandler(BUIacl): ret = func(*args, **kwargs) if ret: break + if not ret: + func = getattr(meta_grants, method) + ret = func(*args, **kwargs) return ret def is_admin(self, username=None): @@ -90,6 +127,16 @@ class ACLhandler(BUIacl): ret = self._iterate_through_loader('is_moderator', username) or False return ret + def is_client_rw(self, username=None, client=None, server=None): + """See :func:`burpui.misc.acl.interface.BUIacl.is_client_rw`""" + ret = self._iterate_through_loader( + 'is_client_rw', + username, + client, + server + ) or False + return ret + def is_client_allowed(self, username=None, client=None, server=None): """See :func:`burpui.misc.acl.interface.BUIacl.is_client_allowed`""" ret = self._iterate_through_loader( @@ -100,6 +147,15 @@ class ACLhandler(BUIacl): ) or False return ret + def is_server_rw(self, username=None, server=None): + """See :func:`burpui.misc.acl.interface.BUIacl.is_server_rw`""" + ret = self._iterate_through_loader( + 'is_server_rw', + username, + server + ) or False + return ret + def is_server_allowed(self, username=None, server=None): """See :func:`burpui.misc.acl.interface.BUIacl.is_server_allowed`""" ret = self._iterate_through_loader( diff --git a/burpui/misc/acl/interface.py b/burpui/misc/acl/interface.py index ac7b8fe8..868455e1 100644 --- a/burpui/misc/acl/interface.py +++ b/burpui/misc/acl/interface.py @@ -7,10 +7,8 @@ .. moduleauthor:: Ziirish """ -from ...utils import make_list - from abc import ABCMeta, abstractmethod, abstractproperty -from six import with_metaclass, iteritems +from six import with_metaclass import logging @@ -22,6 +20,29 @@ class BUIaclLoader(with_metaclass(ABCMeta, object)): logger = logging.getLogger('burp-ui') priority = 0 + add_grant = False + del_grant = False + mod_grant = False + + add_group = False + del_group = False + mod_group = False + + add_group_member = False + del_group_member = False + + add_moderator = False + del_moderator = False + mod_moderator = False + + add_admin = False + del_admin = False + + moderator = None + moderators = [] + + admins = [] + def __init__(self, app=None): """:func:`burpui.misc.acl.interface.BUIaclLoader.__init__` instanciate the loader. @@ -31,12 +52,29 @@ class BUIaclLoader(with_metaclass(ABCMeta, object)): """ pass # pragma: no cover + @abstractmethod + def reload(self): + """Reload the backend""" + return None # pragma: no cover + @abstractproperty @property def acl(self): """Property to retrieve the backend""" return None # pragma: no cover + @abstractproperty + @property + def grants(self): + """Property to retrieve the list of grants""" + return None # pragma: no cover + + @abstractproperty + @property + def groups(self): + """Property to retrieve the list of groups with their members""" + return None # pragma: no cover + class BUIacl(with_metaclass(ABCMeta, object)): """The :class:`burpui.misc.acl.interface.BUIacl` class represents the ACL @@ -100,6 +138,26 @@ class BUIacl(with_metaclass(ABCMeta, object)): """ return [] # pragma: no cover + @abstractmethod + def is_client_rw(self, username=None, client=None, server=None): + """:func:`burpui.misc.acl.interface.BUIacl.is_client_rw` tells us + if a given user has access to a given client on a given server in RW + mode. + + :param username: Username to check + :type username: str + + :param client: Client to check + :type client: str + + :param server: Server to check + :type server: str + + :returns: True if username is granted, otherwise False + :rtype: bool + """ + return False # pragma: no cover + @abstractmethod def is_client_allowed(self, username=None, client=None, server=None): """:func:`burpui.misc.acl.interface.BUIacl.is_client_allowed` tells us @@ -119,6 +177,21 @@ class BUIacl(with_metaclass(ABCMeta, object)): """ return False # pragma: no cover + def is_server_rw(self, username=None, server=None): + """:func:`burpui.misc.acl.interface.BUIacl.is_server_rw` tells us + if a given user has access to a given server in RW mode. + + :param username: Username to check + :type username: str + + :param server: Server to check + :type server: str + + :returns: True if username is granted, otherwise False + :rtype: bool + """ + return False # pragma: no cover + def is_server_allowed(self, username=None, server=None): """:func:`burpui.misc.acl.interface.BUIacl.is_server_allowed` tells us if a given user has access to a given server. @@ -133,115 +206,3 @@ class BUIacl(with_metaclass(ABCMeta, object)): :rtype: bool """ return False # pragma: no cover - - def _merge_data(self, d1, d2): - """Merge data as list or dict recursively avoiding duplicates""" - if not d2: - return d1 - if not d1: - return d2 - if isinstance(d1, list) and isinstance(d2, list): - return list(set(d1 + d2)) - if isinstance(d1, list) and not isinstance(d2, dict): - if d2 in d1: - return d1 - return d1 + [d2] - if isinstance(d2, list) and not isinstance(d1, dict): - if d1 in d2: - return d2 - return d2 + [d1] - if not isinstance(d1, dict) and not isinstance(d2, dict): - if d1 == d2: - return [d1] - else: - return [d1, d2] - - res = d1 - for key2, val2 in iteritems(d2): - if key2 in res: - res[key2] = self._merge_data(val2, res[key2]) - else: - res[key2] = val2 - return res - - def _parse_clients(self, data, mode=None): - agents = clients = [] - advanced = {} - if isinstance(data, list): - if mode: - advanced[mode] = {'clients': data} - return data, agents, advanced - if not isinstance(data, dict): - if mode: - advanced[mode] = {'clients': make_list(data)} - return make_list(data), agents, advanced - for key, val in iteritems(data): - if key in ['agents', 'clients', 'ro', 'rw']: - continue - cl1, ag1, ad1 = self._parse_clients(val) - agents = self._merge_data(agents, ag1) - clients = self._merge_data(clients, cl1) - agents = self._merge_data(agents, key) - advanced = self._merge_data(advanced, ad1) - advanced = self._merge_data(advanced, {key: cl1}) - if mode: - advanced = self._merge_data(advanced, {mode: {key: cl1}}) - - for key in ['clients', 'ro', 'rw']: - md = None - if key in data: - if key in ['ro', 'rw']: - md = key - cl2, ag2, ad2 = self._parse_clients(data[key], md) - agents = self._merge_data(agents, ag2) - clients = self._merge_data(clients, cl2) - advanced = self._merge_data(advanced, ad2) - - if 'agents' in data: - ag3, cl3, ad3 = self._parse_agents(data['agents']) - agents = self._merge_data(agents, ag3) - clients = self._merge_data(clients, cl3) - advanced = self._merge_data(advanced, ad3) - - return make_list(clients), make_list(agents), advanced - - def _parse_agents(self, data, mode=None): - agents = clients = [] - advanced = {} - if isinstance(data, list): - if mode: - advanced[mode] = {'agents': data} - return data, clients, advanced - if not isinstance(data, dict): - if mode: - advanced[mode] = {'agents': make_list(data)} - return make_list(data), clients, advanced - for key, val in iteritems(data): - if key in ['agents', 'clients', 'ro', 'rw']: - continue - cl1, ag1, ad1 = self._parse_clients(data) - agents = self._merge_data(agents, ag1) - clients = self._merge_data(clients, cl1) - agents = self._merge_data(agents, key) - advanced = self._merge_data(advanced, ad1) - advanced = self._merge_data(advanced, {key: cl1}) - if mode: - advanced = self._merge_data(advanced, {mode: {key: cl1}}) - - for key in ['agents', 'ro', 'rw']: - md = None - if key in data: - if key in ['ro', 'rw']: - md = key - ag2, cl2, ad2 = self._parse_agents(data[key], md) - agents = self._merge_data(agents, ag2) - clients = self._merge_data(clients, cl2) - advanced = self._merge_data(advanced, ad2) - - if 'clients' in data: - cl3, ag3, ad3 = self._parse_clients(data['clients']) - agents = self._merge_data(agents, ag3) - clients = self._merge_data(clients, cl3) - advanced = self._merge_data(advanced, ad3) - - return make_list(agents), make_list(clients), advanced diff --git a/burpui/misc/acl/meta.py b/burpui/misc/acl/meta.py new file mode 100644 index 00000000..d3d613b5 --- /dev/null +++ b/burpui/misc/acl/meta.py @@ -0,0 +1,553 @@ +# -*- coding: utf8 -*- +""" +.. module:: burpui.misc.acl.meta + :platform: Unix + :synopsis: Burp-UI ACL meta definitions. + +.. moduleauthor:: Ziirish + +""" +from .interface import BUIacl +from ...utils import make_list + +from six import iteritems, itervalues + +import re +import json +import fnmatch + + +class BUImetaGrant(object): + + def _merge_data(self, d1, d2): + """Merge data as list or dict recursively avoiding duplicates""" + if not d1 and not d2: + return [] + if not d2: + return d1 + if not d1: + return d2 + if isinstance(d1, list) and isinstance(d2, list): + return list(set(d1 + d2)) + if isinstance(d1, list) and not isinstance(d2, dict): + if d2 in d1: + return d1 + return d1 + make_list(d2) + if isinstance(d2, list) and not isinstance(d1, dict): + if d1 in d2: + return d2 + return d2 + make_list(d1) + if not isinstance(d1, dict) and not isinstance(d2, dict): + if d1 == d2: + return make_list(d1) + else: + return [d1, d2] + + res = d1 + for key2, val2 in iteritems(d2): + if key2 in res: + res[key2] = self._merge_data(val2, res[key2]) + else: + res[key2] = val2 + return res + + def _parse_clients(self, data, mode=None): + agents = clients = [] + advanced = {} + if isinstance(data, list): + if mode: + advanced[mode] = {'clients': data} + return data, agents, advanced + if not isinstance(data, dict): + if mode: + advanced[mode] = {'clients': make_list(data)} + return make_list(data), agents, advanced + for key, val in iteritems(data): + if key in ['agents', 'clients', 'ro', 'rw']: + continue + cl1, ag1, ad1 = self._parse_clients(val) + agents = self._merge_data(agents, ag1) + clients = self._merge_data(clients, cl1) + agents = self._merge_data(agents, key) + advanced = self._merge_data(advanced, ad1) + advanced = self._merge_data(advanced, {key: cl1}) + if mode: + advanced = self._merge_data(advanced, {mode: {key: cl1}}) + + for key in ['clients', 'ro', 'rw']: + md = None + if key in data: + if key in ['ro', 'rw']: + md = key + cl2, ag2, ad2 = self._parse_clients(data[key], md) + agents = self._merge_data(agents, ag2) + clients = self._merge_data(clients, cl2) + advanced = self._merge_data(advanced, ad2) + + if 'agents' in data: + ag3, cl3, ad3 = self._parse_agents(data['agents']) + agents = self._merge_data(agents, ag3) + clients = self._merge_data(clients, cl3) + advanced = self._merge_data(advanced, ad3) + + return make_list(clients), make_list(agents), advanced + + def _parse_agents(self, data, mode=None): + agents = clients = [] + advanced = {} + if isinstance(data, list): + if mode: + advanced[mode] = {'agents': data} + return data, clients, advanced + if not isinstance(data, dict): + if mode: + advanced[mode] = {'agents': make_list(data)} + return make_list(data), clients, advanced + for key, val in iteritems(data): + if key in ['agents', 'clients', 'ro', 'rw']: + continue + cl1, ag1, ad1 = self._parse_clients(data) + agents = self._merge_data(agents, ag1) + clients = self._merge_data(clients, cl1) + agents = self._merge_data(agents, key) + advanced = self._merge_data(advanced, ad1) + advanced = self._merge_data(advanced, {key: cl1}) + if mode: + advanced = self._merge_data(advanced, {mode: {key: cl1}}) + + for key in ['agents', 'ro', 'rw']: + md = None + if key in data: + if key in ['ro', 'rw']: + md = key + ag2, cl2, ad2 = self._parse_agents(data[key], md) + agents = self._merge_data(agents, ag2) + clients = self._merge_data(clients, cl2) + advanced = self._merge_data(advanced, ad2) + + if 'clients' in data: + cl3, ag3, ad3 = self._parse_clients(data['clients']) + agents = self._merge_data(agents, ag3) + clients = self._merge_data(clients, cl3) + advanced = self._merge_data(advanced, ad3) + + return make_list(agents), make_list(clients), advanced + + +class BUIgrantHandler(BUImetaGrant, BUIacl): + """This class is here to handle grants in a generic way. + It will automatically merge grants from various backends that register to it + """ + _id = 1 + _gp_admin_name = '@BUIADMINRESERVED' + _gp_moderator_name = '@moderator' + + _grants = {} + _groups = {} + + _parsed_grants = [] + + _clients_cache = {} + _agents_cache = {} + _advanced_cache = {} + + _options = {} + _backends = {} + + @property + def id(self): + """current handler id, used to detect configuration changes""" + return self._id + + @property + def grants(self): + """grants managed by our handler""" + return self._grants + + @property + def groups(self): + """groups managed by our handler""" + return self._groups + + @property + def options(self): + """options of our ACL engine""" + return self._options + + @options.setter + def options(self, value): + """set the options of our engine""" + self._options = value + if self._options.get('legacy'): + self._options['extended'] = False + + def changed(self, sid): + """detect a configuration change""" + return sid != self._id + + def reset(self, reset_from): + """a configuration change occurred, we reload our grants and groups""" + self._grants.clear() + self._groups.clear() + self._parsed_grants = [] + self._clients_cache.clear() + self._agents_cache.clear() + self._advanced_cache.clear() + self._id += 1 + for name, backend in iteritems(self._backends): + if name == reset_from: + continue + backend.reload() + + def opt(self, key, default=False): + """access a given option""" + if key not in self.options: + return default + return self.options.get(key) + + def register_backend(self, name, backend): + """register a new ACL backend + + :param name: Backend name + :type name: str + + :param backend: ACL Backend + :type backend: :class:`burpui.misc.acl.interface.BUIaclLoader` + """ + self._backends[name] = backend + + def set_grant(self, name, grant): + """parse and set the given grants""" + if name in self._grants: + return self._grants[name].add_grants(grant) + self._grants[name] = BUIaclGrant(name, grant) + return self._grants[name].grants + + def set_group(self, name, members): + """parse and set the given group""" + if name in self._groups: + return self._groups[name].add_members(members) + self._groups[name] = BUIaclGroup(name, members) + return self._groups[name].members + + def set_admin(self, admins): + """parse and set the admins""" + self.set_group(self._gp_admin_name, admins) + + def set_moderator(self, moderators): + """parse and set the moderators""" + self.set_group(self._gp_moderator_name, moderators) + + def set_moderator_grants(self, grants): + """parse and set the moderators grants""" + self.set_grant(self._gp_moderator_name, grants) + + def get_member_groups(self, member): + groups = [] + for group in itervalues(self._groups): + if group.is_member(member): + groups.append(group.name) + return groups + + def _extract_grants(self, username): + if username not in self._parsed_grants: + + if username in self.grants: + grants = self.grants[username].grants + else: + grants = [] + + clients, agents, advanced = self._parse_clients(grants) + self._clients_cache[username] = clients + self._agents_cache[username] = agents + self._advanced_cache[username] = advanced + + def __merge_grants_with(grp): + if grp not in self._parsed_grants: + self._extract_grants(grp) + self._clients_cache[username] = self._merge_data( + self._clients_cache[username], + self._clients_cache.get(grp, []) + ) + self._agents_cache[username] = self._merge_data( + self._agents_cache[username], + self._agents_cache.get(grp, []) + ) + self._advanced_cache[username] = self._merge_data( + self._advanced_cache[username], + self._advanced_cache.get(grp, []) + ) + + # moderator is also a group + for gname, group in iteritems(self.groups): + # no grants need to be parsed for admins + if gname == self._gp_admin_name: + continue + if group.is_member(username) and gname != username: + __merge_grants_with(gname) + + self._parsed_grants.append(username) + + def _extract_clients(self, username): + if username not in self._parsed_grants: + self._extract_grants(username) + return self._clients_cache.get(username, []) + + def _extract_agents(self, username): + if username not in self._parsed_grants: + self._extract_grants(username) + return self._agents_cache.get(username, []) + + def _extract_advanced(self, username): + if username not in self._parsed_grants: + self._extract_grants(username) + return self._advanced_cache.get(username, {}) + + def _extract_advanced_mode(self, username, mode, kind): + if username not in self._parsed_grants: + self._extract_grants(username) + return self._advanced_cache.get(username, {}).get(mode, {}).get(kind, []) + + def _client_match(self, username, client): + clients = self._extract_clients(username) + if not clients: + return None + + if self.opt('extended'): + for exp in clients: + regex = fnmatch.translate(exp) + if re.match(regex, client): + return exp + return False + else: + return client if client in clients else False + + def _server_match(self, username, server): + servers = self._extract_agents(username) + if not servers: + return None + + if self.opt('extended'): + for exp in servers: + regex = fnmatch.translate(exp) + if re.match(regex, server): + return exp + return False + else: + return server if server in servers else False + + # implement BUIacl methods + + def is_admin(self, username): + """See :func:`burpui.misc.acl.interface.BUIacl.is_admin`""" + return self._gp_admin_name in self._groups and \ + self._groups[self._gp_admin_name].is_member(username) + + def is_moderator(self, username): + """See :func:`burpui.misc.acl.interface.BUIacl.is_moderator`""" + return self._gp_moderator_name in self._groups and \ + self._groups[self._gp_moderator_name].is_member(username) + + def is_client_rw(self, username=None, client=None, server=None): + """See :func:`burpui.misc.acl.interface.BUIacl.is_client_rw`""" + if not username or not client: # pragma: no cover + return False + + is_admin = self.is_admin(username) + + if self.is_client_allowed(username, client, server): + # legacy mode: assume rw for everyone + if self.opt('legacy'): + return True + client_match = self._client_match(username, client) + advanced = self._extract_advanced(username) + + if not client_match and username == client: + client_match = username + + if server: + server_match = self._server_match(username, server) + + if not server_match and not client_match: + return is_admin or self.opt('assume_granted') + + # the whole agent is rw and we did not find explicit entry for + # client_match + if client_match is False: + if server_match in advanced.get('rw', {}) or \ + server_match in advanced.get('rw', {}).get('agents', []): + return True + if server in advanced.get('rw', {}) or \ + server in advanced.get('rw', {}).get('agents', []): + return True + + if server_match and \ + (server_match in advanced.get('ro', {}) or + server_match in advanced.get('ro', {}).get('agents', [])): + # the agent is ro, but the client is explicitly defined as rw + if client_match and \ + (client_match not in advanced.get('rw', {}).get(server_match, []) or + client_match not in advanced.get('rw', {}).get('clients', [])): + return True + + rw_clients = advanced.get('rw', {}).get('clients', []) + if client_match and \ + client_match in rw_clients: + return True + + if client and \ + client in rw_clients: + return True + + if self.opt('legacy'): + return True + return is_admin or self.opt('assume_granted') + + def is_client_allowed(self, username=None, client=None, server=None): + """See :func:`burpui.misc.acl.interface.BUIacl.is_client_allowed`""" + if not username or not client: # pragma: no cover + return False + + is_admin = self.is_admin(username) + client_match = self._client_match(username, client) + + if not client_match and username == client: + client_match = username + + if server: + server_match = self._server_match(username, server) + if server_match is not None or self.opt('legacy'): + if not server_match: + return is_admin + + advanced = self._extract_advanced(username) + + if not client_match and server_match not in advanced and \ + (server_match in self._extract_advanced_mode(username, 'ro', 'agents') or + server_match in self._extract_advanced_mode(username, 'rw', 'agents')): + return True + + advanced = advanced.get(server_match, advanced.get(server, [])) + if client_match not in advanced and client not in advanced: + return is_admin + + return client_match is not False or is_admin + + def is_server_rw(self, username=None, server=None): + """See :func:`burpui.misc.acl.interface.BUIacl.is_server_rw`""" + if not username or not server: # pragma: no cover + return False + + is_admin = self.is_admin(username) + if self.is_server_allowed(username, server): + server_match = self._server_match(username, server) + if not server_match: + return self.is_admin or self.opt('assume_granted') + + advanced = self._extract_advanced(username) + + if server_match in advanced.get('rw', {}).get('agents', []): + return True + + if self.opt('legacy'): + return True + return is_admin or self.opt('assume_granted') + + def is_server_allowed(self, username=None, server=None): + """See :func:`burpui.misc.acl.interface.BUIacl.is_server_allowed`""" + if not username or not server: + return False + + server_match = self._server_match(username, server) + is_admin = self.is_admin(username) + + if server_match is None and self.opt('legacy'): + server_match = False + + return server_match is not False or is_admin + + +class BUIaclGroup(object): + """The :class:`burpui.misc.acl.interface.BUIaclGroup` class is used to + represent a Group""" + + def __init__(self, name, members=None): + self._name = name + self._set_members(members) + + def _parse_members(self, members): + # we support only lists + if ',' in members and not isinstance(members, list): + parsed = [x.strip() for x in members.split(',')] + else: + parsed = make_list(members) + return parsed + + def _set_members(self, members): + self._members = set(self._parse_members(members)) + + def add_members(self, new_members): + new_members = self._parse_members(new_members) + self._members = self._members | set(new_members) + return new_members + + def del_members(self, members_remove): + members_remove = self._parse_members(members_remove) + self._members = self._members - set(members_remove) + + def is_member(self, member): + return member in self._members + + @property + def name(self): + if self._name and any(self._name.startswith(x) for x in ['@', '+']): + return str(self._name[1:]) + return self._name + + @property + def members(self): + return list(self._members) + + +class BUIaclGrant(BUImetaGrant): + """The :class:`burpui.misc.acl.interface.BUIaclGrant` class is used to + represent a Grant""" + + def __init__(self, name, grants): + self._name = name + self._grants = self._parse_grants(grants) + + def _parse_grants(self, grants): + try: + ret = json.loads(grants) + except (ValueError, TypeError): + # ignore mal-formatted json + if any([x in grants for x in ['{', '}', '[', ']']]): + ret = None + elif ',' in grants: + ret = [x.rstrip() for x in grants.split(',')] + else: + ret = make_list(grants) + return ret + + @property + def name(self): + if self._name and any(self._name.startswith(x) for x in ['@', '+']): + return str(self._name[1:]) + return self._name + + @property + def grants(self): + return self._grants + + @property + def grants_raw(self): + return json.dumps(self._grants) + + def add_grants(self, grants): + parsed = self._parse_grants(grants) + self._grants = self._merge_data(self._grants, parsed) + return parsed + + +meta_grants = BUIgrantHandler() diff --git a/burpui/misc/auth/handler.py b/burpui/misc/auth/handler.py index bd4756ba..4e4e28f2 100644 --- a/burpui/misc/auth/handler.py +++ b/burpui/misc/auth/handler.py @@ -12,6 +12,8 @@ from six import iteritems from collections import OrderedDict from flask_login import AnonymousUserMixin +ACL_METHODS = BUIacl.__abstractmethods__ + class UserAuthHandler(BUIhandler): """See :class:`burpui.misc.auth.interface.BUIhandler`""" @@ -47,7 +49,7 @@ class UserAuthHandler(BUIhandler): except: import traceback self.errors[name] = traceback.format_exc() - backends.sort(key=lambda x: x.priority, reverse=True) + backends.sort(key=lambda x: getattr(x, 'priority', -1), reverse=True) if not backends: raise ImportError( 'No backend found for \'{}\':\n{}'.format(self.app.auth, @@ -94,45 +96,102 @@ class UserAuthHandler(BUIhandler): if name in self.users: del self.users[name] + @property + def loader(self): + return None + + +class ProxyACLCall(object): + """Class that actually calls the ACL method""" + def __init__(self, acl, username, method): + """ + :param acl: ACL to use + :type acl: :class:`burpui.misc.acl.interface.BUIacl` + + :param username: username to check ACL for + :type username: str + + :param method: Name of the method to proxify + :type method: str + """ + self.acl = acl + self.username = username + self.method = method + + def __call__(self, *args, **kwargs): + """This is where the proxy call (and the magic) occurs""" + # retrieve the original function prototype + proto = getattr(BUIacl, self.method) + args_name = list(proto.__code__.co_varnames) + # skip self + args_name.pop(0) + # skip username + args_name.pop(0) + # we transform unnamed arguments to named ones + # example: + # def my_function(toto, tata=None, titi=None): + # + # x = my_function('blah', titi='blih') + # + # => {'toto': 'blah', 'titi': 'blih'} + encoded_args = { + 'username': self.username + } + for idx, opt in enumerate(args): + encoded_args[args_name[idx]] = opt + encoded_args.update(kwargs) + + func = getattr(self.acl, self.method) + return func(**encoded_args) + class ACLproxy(BUIacl): + foreign = ACL_METHODS + BUIacl.__abstractmethods__ = frozenset() + def __init__(self, acl, username): self.acl = acl self.username = username - def is_admin(self): - if not self.acl: - return True - return self.acl.is_admin(self.username) + def __getattribute__(self, name): + # always return this value because we need it and if we don't do that + # we'll end up with an infinite loop + if name == 'foreign': + return object.__getattribute__(self, name) + # now we can retrieve the 'foreign' list and know if the object called + # needs to be "proxyfied" + if name in self.foreign: + if self.acl: + return ProxyACLCall(self.acl, self.username, name) + # no ACL, assume true + return ProxyTrue() + return object.__getattribute__(self, name) - def is_moderator(self): - if not self.acl: - return True - return self.acl.is_moderator(self.username) - def is_client_allowed(self, client, server=None): - if not self.acl: - return True - return self.acl.is_client_allowed(self.username, client, server) +class ProxyTrue(object): + def __call__(self, *args, **kwargs): + return True - def is_server_allowed(self, server): - if not self.acl: - return True - return self.acl.is_server_allowed(self.username, server) + +class ProxyFalse(object): + def __call__(self, *args, **kwargs): + return False class ACLanon(BUIacl): - def is_admin(self): - return False + foreign = ACL_METHODS + BUIacl.__abstractmethods__ = frozenset() - def is_moderator(self): - return False - - def is_client_allowed(self, client, server=None): - return False - - def is_server_allowed(self, server): - return False + def __getattribute__(self, name): + # always return this value because we need it and if we don't do that + # we'll end up with an infinite loop + if name == 'foreign': + return object.__getattribute__(self, name) + # now we can retrieve the 'foreign' list and know if the object called + # needs to be "proxyfied" + if name in self.foreign: + return ProxyFalse() + return object.__getattribute__(self, name) class BUIanon(AnonymousUserMixin): @@ -150,6 +209,10 @@ class BUIanon(AnonymousUserMixin): def is_admin(self): return False + @property + def is_moderator(self): + return False + class UserHandler(BUIuser): """See :class:`burpui.misc.auth.interface.BUIuser`""" @@ -180,14 +243,13 @@ class UserHandler(BUIuser): continue res = user.get_id() if res: + self.real = user self.active = True self.name = res self.back = back break self._acl = ACLproxy(self.app.acl, self.name) - # language may change upon login - self._store_lang() # now load the available prefs self._load_prefs() @@ -195,31 +257,26 @@ class UserHandler(BUIuser): def acl(self): return self._acl + @property + def is_admin(self): + return self.acl.is_admin() + + @property + def is_moderator(self): + return self.acl.is_moderator() + def _load_prefs(self): session['login'] = self.name if self.app.config['WITH_SQL']: from ...models import Pref prefs = Pref.query.filter_by(user=self.name).all() for pref in prefs: + if pref.key == 'language': + continue if hasattr(self, pref.key): setattr(self, pref.key, pref.value) session[pref.key] = pref.value - def _store_lang(self): - if self.app.config['WITH_SQL'] and self.language: - from ...ext.sql import db - from ...models import Pref - pref = Pref.query.filter_by(user=self.name, key='language').first() - if pref: - pref.value = self.language - else: - pref = Pref(self.name, 'language', self.language) - db.session.add(pref) - try: - db.session.commit() - except: - db.session.rollback() - def refresh_session(self): self.authenticated = session.get('authenticated', False) self.language = session.get('language', None) diff --git a/burpui/misc/auth/interface.py b/burpui/misc/auth/interface.py index 2b36ee12..db13e5e1 100644 --- a/burpui/misc/auth/interface.py +++ b/burpui/misc/auth/interface.py @@ -8,7 +8,7 @@ """ from flask_login import UserMixin -from abc import ABCMeta, abstractmethod +from abc import ABCMeta, abstractmethod, abstractproperty from six import with_metaclass import logging @@ -59,6 +59,7 @@ class BUIhandler(with_metaclass(ABCMeta, object)): """ pass + @abstractproperty @property def loader(self): return None @@ -70,6 +71,7 @@ class BUIuser(with_metaclass(ABCMeta, UserMixin)): """ backend = None admin = True + moderator = True @abstractmethod def login(self, passwd=None): @@ -106,6 +108,15 @@ class BUIuser(with_metaclass(ABCMeta, UserMixin)): """ return self.admin + @property + def is_moderator(self): + """ + If no ACL engine is loaded, every logged-in user will be granted + moderator rights + :returns: True if the user is moderator, otherwise False + """ + return self.moderator + def __str__(self): msg = UserMixin.__str__(self) return '{} (id: {}, admin: {}, authenticated: {}, active: {})'.format( diff --git a/burpui/misc/auth/ldap.py b/burpui/misc/auth/ldap.py index 402a3b1c..25ed58f7 100644 --- a/burpui/misc/auth/ldap.py +++ b/burpui/misc/auth/ldap.py @@ -205,10 +205,10 @@ class LdapLoader(BUIloader): :returns: True if bind was successful, otherwise False """ try: - with Connection(self.server, user='{0}'.format(dn), password=passwd, raise_exceptions=True, auto_bind=self.auto_bind, authentication=SIMPLE) as l: - self.logger.debug('LDAP Connection = {0}'.format(str(l))) + with Connection(self.server, user='{0}'.format(dn), password=passwd, raise_exceptions=True, auto_bind=self.auto_bind, authentication=SIMPLE) as con: + self.logger.debug('LDAP Connection = {0}'.format(str(con))) self.logger.info('Bound as user: {0}'.format(dn)) - return l.bind() + return con.bind() except Exception as e: self.logger.error('Failed to authenticate user: {0}, {1}'.format(dn, str(e))) diff --git a/burpui/misc/backend/burp1.py b/burpui/misc/backend/burp1.py index 3e5e6cb5..ba629ba8 100644 --- a/burpui/misc/backend/burp1.py +++ b/burpui/misc/backend/burp1.py @@ -680,11 +680,48 @@ class Burp(BUIbackend): cli['last'] = int(spl[2]) else: spl = infos.split('\t') - cli['last'] = int(spl[len(spl) - 2]) + cli['last'] = int((spl[-1].split())[-1]) cli['last'] = utc_to_local(cli['last']) res.append(cli) return res + def get_client_status(self, name=None, agent=None): + """See :func:`burpui.misc.backend.interface.BUIbackend.get_client_status`""" + cli = {} + filemap = self.status('c:{0}\n'.format(name)) + for line in filemap: + if not re.match('^{0}\t'.format(name), line): + continue + regex = re.compile(r'\s*(\S+)\s+\d\s+(\S)\s+(.+)') + match = regex.search(line) + cli['state'] = self.states[match.group(2)] + infos = match.group(3) + if cli['state'] in ['running']: + regex = re.compile(r'\s*(\S+)') + reg = regex.search(infos) + phase = reg.group(0) + if phase and phase in self.states: + cli['phase'] = self.states[phase] + else: + cli['phase'] = 'unknown' + cli['last'] = 'now' + counters = self.get_counters(name) + if 'percent' in counters: + cli['percent'] = counters['percent'] + else: + cli['percent'] = 0 + elif infos == "0": + cli['last'] = 'never' + elif re.match(r'^\d+\s\d+\s\d+$', infos): + spl = infos.split() + cli['last'] = int(spl[2]) + else: + spl = infos.split('\t') + cli['last'] = int((spl[-1].split())[-1]) + cli['last'] = utc_to_local(cli['last']) + break + return cli + def get_client(self, name=None, agent=None): """See :func:`burpui.misc.backend.interface.BUIbackend.get_client`""" return self.get_client_filtered(name) @@ -857,23 +894,27 @@ class Burp(BUIbackend): if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) full_reg = u'' + + def _escape(s): + return re.sub(r"[(){}\[\].*?|^$\\+-]", r"\\\g<0>", s) + for restore in flist['restore']: reg = u'' if restore['folder'] and restore['key'] != '/': - reg += '^' + re.escape(restore['key']) + '/|' + reg += '^' + _escape(restore['key']) + '/|' else: - reg += '^' + re.escape(restore['key']) + '$|' - full_reg += reg + reg += '^' + _escape(restore['key']) + '$|' + full_reg += to_unicode(reg) - cmd = [self.burpbin, '-C', quote(name), '-a', 'r', '-b', quote(str(backup)), '-r', full_reg.rstrip('|'), '-d', tmpdir] + cmd = [self.burpbin, '-C', quote(name), '-a', 'r', '-b', quote(str(backup)), '-r', full_reg.rstrip('|').replace(r"\n", r"\\n"), '-d', tmpdir] if password: if not self.burpconfcli: return None, 'No client configuration file specified' tmpdesc = os.fdopen(tmphandler, 'wb+') - with open(self.burpconfcli) as fileobj: + with open(self.burpconfcli, 'rb') as fileobj: shutil.copyfileobj(fileobj, tmpdesc) - tmpdesc.write('encryption_password = {}\n'.format(sanitize_string(password))) + tmpdesc.write(to_bytes('encryption_password = {}\n'.format(sanitize_string(password)))) tmpdesc.close() cmd.append('-c') cmd.append(tmpfile) @@ -959,7 +1000,7 @@ class Burp(BUIbackend): return [] return self.parser.read_server_conf(conf) - def store_conf_cli(self, data, client=None, conf=None, agent=None): + def store_conf_cli(self, data, client=None, conf=None, template=False, agent=None): """See :func:`burpui.misc.backend.interface.BUIbackend.store_conf_cli`""" if not self.parser: return [] @@ -967,7 +1008,7 @@ class Burp(BUIbackend): conf = unquote(conf) except: pass - return self.parser.store_client_conf(data, client, conf) + return self.parser.store_client_conf(data, client, conf, template) def store_conf_srv(self, data, conf=None, agent=None): """See :func:`burpui.misc.backend.interface.BUIbackend.store_conf_srv`""" diff --git a/burpui/misc/backend/burp2.py b/burpui/misc/backend/burp2.py index ac6b5519..eef8e225 100644 --- a/burpui/misc/backend/burp2.py +++ b/burpui/misc/backend/burp2.py @@ -13,6 +13,7 @@ import time import subprocess import sys import json +import datetime from select import select from six import iteritems, viewkeys @@ -33,6 +34,23 @@ BURP_LIST_BATCH = '2.0.48' BURP_STATUS_FORMAT_V2 = '2.1.10' BURP_REVERSE_COUNTERS = '2.1.6' +try: + import gevent + from gevent.lock import RLock + + G_LOCK = RLock() + WITH_GEVENT = True +except ImportError: + class DummyLock(object): + def __enter__(self): + return self + + def __exit__(*x): + pass + + G_LOCK = DummyLock() + WITH_GEVENT = False + # Some functions are the same as in Burp1 backend class Burp(Burp1): @@ -54,6 +72,10 @@ class Burp(Burp1): _vers = 2 # cache to store the guessed OS _os_cache = {} + # cache status results + _status_cache = {} + _last_status_cleanup = datetime.datetime.now() + _time_to_cache = datetime.timedelta(seconds=3) def __init__(self, server=None, conf=None): """ @@ -154,8 +176,6 @@ class Burp(Burp1): shell=False, bufsize=0 ) - # wait a little bit in case the process dies on a network error - time.sleep(0.5) if not self._proc_is_alive(): details = u'' if verbose: @@ -191,7 +211,7 @@ class Burp(Burp1): if not self.server_version: if 'logline' in jso: ret = re.search( - r'^Server version: (\d+\.\d+\.\d+)$', + r'^Server version: (\d+\.\d+\.\d+).*$', jso['logline'] ) if ret: @@ -272,31 +292,48 @@ class Burp(Burp1): # the os throws an exception if there is no data or timeout self.logger.warning(str(exp)) self._kill_burp() - break + return None return jso - def status(self, query='c:\n', timeout=None, agent=None): + def _cleanup_cache(self): + now = datetime.datetime.now() + if now - self._last_status_cleanup > self._time_to_cache: + self._status_cache.clear() + self._last_status_cleanup = now + + def status(self, query='c:\n', timeout=None, cache=True, agent=None): """See :func:`burpui.misc.backend.interface.BUIbackend.status`""" try: timeout = timeout or self.timeout query = sanitize_string(query.rstrip()) self.logger.info("query: '{}'".format(query)) query = '{0}\n'.format(query) - if not self._proc_is_alive(): - self._spawn_burp() - _, write, _ = select([], [self.proc.stdin], [], self.timeout) - if self.proc.stdin not in write: - raise TimeoutError('Write operation timed out') - self.proc.stdin.write(to_bytes(query)) - jso = self._read_proc_stdout(timeout) - if self._is_warning(jso): - self.logger.warning(jso['warning']) - self.logger.debug('Nothing interesting to return') - return None + with G_LOCK: + self._cleanup_cache() + # return cached results + if cache and query in self._status_cache: + return self._status_cache[query] - self.logger.debug('=> {}'.format(jso)) - return jso + if not self._proc_is_alive(): + self._spawn_burp() + + _, write, _ = select([], [self.proc.stdin], [], self.timeout) + if self.proc.stdin not in write: + raise TimeoutError('Write operation timed out') + self.proc.stdin.write(to_bytes(query)) + jso = self._read_proc_stdout(timeout) + if self._is_warning(jso): + self.logger.warning(jso['warning']) + self.logger.debug('Nothing interesting to return') + return None + + self.logger.debug('=> {}'.format(jso)) + + if cache: + self._status_cache[query] = jso + + return jso except TimeoutError as exp: msg = 'Cannot send command: {}'.format(str(exp)) self.logger.error(msg) @@ -327,14 +364,6 @@ class Burp(Burp1): return ret if 'backup_stats' in logs: ret = self._parse_backup_stats(number, client, forward) - # TODO: support clients that were upgraded to 2.x - # else: - # cl = None - # if forward: - # cl = client - - # f = self.status('c:{0}:b:{1}:f:log.gz\n'.format(client, number)) - # ret = self._parse_backup_log(f, number, cl) ret['encrypted'] = False if 'files_enc' in ret and ret['files_enc']['total'] > 0: @@ -417,11 +446,6 @@ class Burp(Burp1): 'total': 'scanned', 'scanned': 'scanned', } - # Prior burp-2.1.6 some counters are reversed - # See https://github.com/grke/burp/commit/adeb3ad68477303991a393fa7cd36bc94ff6b429 - if self.server_version and self.server_version < BURP_REVERSE_COUNTERS: - counts['changed'] = 'same' - counts['unchanged'] = 'changed' single = [ 'time_start', 'time_end', @@ -431,8 +455,7 @@ class Burp(Burp1): 'bytes' ] query = self.status( - 'c:{0}:b:{1}:l:backup_stats\n'.format(client, number), - agent=agent + 'c:{0}:b:{1}:l:backup_stats\n'.format(client, number) ) if not query: return ret @@ -508,7 +531,7 @@ class Burp(Burp1): else: if not name or name not in self.running: return ret - query = self.status('c:{0}\n'.format(name)) + query = self.status('c:{0}\n'.format(name), cache=False) # check the status returned something if not query: return ret @@ -559,16 +582,27 @@ class Burp(Burp1): except KeyError: return cntr - for counter in backup['counters']: + for counter in backup.get('counters', {}): name = translate(counter['name']) if counter['name'] not in single: - ret[name] = [ - counter['count'], - counter['changed'], - counter['same'], - counter['deleted'], - counter['scanned'] - ] + # Prior burp-2.1.6 some counters are reversed + # See https://github.com/grke/burp/commit/adeb3ad68477303991a393fa7cd36bc94ff6b429 + if self.server_version and self.server_version < BURP_REVERSE_COUNTERS: + ret[name] = [ + counter['count'], + counter['same'], # reversed + counter['changed'], # reversed + counter['deleted'], + counter['scanned'] + ] + else: + ret[name] = [ + counter['count'], + counter['changed'], + counter['same'], + counter['deleted'], + counter['scanned'] + ] else: ret[name] = counter['count'] @@ -576,7 +610,7 @@ class Burp(Burp1): ret['phase'] = backup['phase'] else: for phase in phases: - if phase in backup['flags']: + if phase in backup.get('flags', []): ret['phase'] = phase break @@ -734,13 +768,7 @@ class Burp(Burp1): cli['state'] = self._status_human_readable(client['run_status']) infos = client['backups'] if cli['state'] in ['running']: - cli['phase'] = client['phase'] cli['last'] = 'now' - counters = self.get_counters(cli['name']) - if 'percent' in counters: - cli['percent'] = counters['percent'] - else: - cli['percent'] = 0 elif not infos: cli['last'] = 'never' else: @@ -749,6 +777,44 @@ class Burp(Burp1): ret.append(cli) return ret + def get_client_status(self, name=None, agent=None): + """See :func:`burpui.misc.backend.interface.BUIbackend.get_client_status`""" + ret = {} + if not name: + return ret + query = self.status('c:{0}\n'.format(name)) + if not query: + return ret + try: + client = query['clients'][0] + except (KeyError, IndexError): + self.logger.warning('Client not found') + return ret + ret['state'] = self._status_human_readable(client['run_status']) + infos = client['backups'] + if ret['state'] in ['running']: + try: + ret['phase'] = client['phase'] + except KeyError: + for child in client.get('children', []): + if 'action' in child and child['action'] == 'backup': + ret['phase'] = child['phase'] + break + if name not in self.running: + self.is_one_backup_running() + counters = self.get_counters(name) + if 'percent' in counters: + ret['percent'] = counters['percent'] + else: + ret['percent'] = 0 + ret['last'] = 'now' + elif not infos: + ret['last'] = 'never' + else: + infos = infos[0] + ret['last'] = infos['timestamp'] + return ret + def get_client(self, name=None, agent=None): """See :func:`burpui.misc.backend.interface.BUIbackend.get_client`""" return self.get_client_filtered(name) @@ -763,13 +829,14 @@ class Burp(Burp1): return ret try: backups = query['clients'][0]['backups'] - except KeyError: + except (KeyError, IndexError): self.logger.warning('Client not found') return ret - for cpt, backup in enumerate(backups): + threads = [] + for idx, backup in enumerate(backups): # skip the first elements if we are in a page if page and page > 1 and limit > 0: - if cpt < (page - 1) * limit: + if idx < (page - 1) * limit: continue back = {} # skip running backups since data will be inconsistent @@ -787,31 +854,45 @@ class Burp(Burp1): # skip backups after "end" if end and backup['timestamp'] > end: continue - log = self.get_backup_logs(backup['number'], name) - try: - back['encrypted'] = log['encrypted'] + + def __get_log(client, bkp, res): + log = self.get_backup_logs(bkp['number'], client) try: - back['received'] = log['received'] - except KeyError: - back['received'] = 0 - try: - back['size'] = log['totsize'] - except KeyError: - back['size'] = 0 - back['end'] = log['end'] - # override date since the timestamp is odd - back['date'] = log['start'] - ret.append(back) - except Exception: - self.logger.warning('Unable to parse logs') - pass + res['encrypted'] = log['encrypted'] + try: + res['received'] = log['received'] + except KeyError: + res['received'] = 0 + try: + res['size'] = log['totsize'] + except KeyError: + res['size'] = 0 + res['end'] = log['end'] + # override date since the timestamp is odd + res['date'] = log['start'] + except Exception: + self.logger.warning('Unable to parse logs') + return None + return res + + if WITH_GEVENT: + threads.append(gevent.spawn(__get_log, name, backup, back)) + else: + with_log = __get_log(name, backup, back) + if with_log: + ret.append(with_log) + # stop after "limit" elements if page and page > 1 and limit > 0: - if cpt >= page * limit: + if idx >= page * limit: break - elif limit > 0 and cpt >= limit: + elif limit > 0 and idx >= limit: break + if WITH_GEVENT: + gevent.joinall(threads) + ret = [x.value for x in threads] + # Here we need to reverse the array so the backups are sorted by num # ASC ret.reverse() @@ -908,11 +989,16 @@ class Burp(Burp1): ret = [] if not client: return ret - query = self.status('c:{0}\n'.format(client)) + # micro optimization since the status results are cached in memory for a + # couple seconds, using the same global query and iterating over it + # will be more efficient than filtering burp-side + query = self.status('c:\n') if not query: return ret try: - return query['clients'][0]['labels'] + for cli in query['clients']: + if cli['name'] == client: + return cli['labels'] except KeyError: return ret diff --git a/burpui/misc/backend/interface.py b/burpui/misc/backend/interface.py index a295a098..6554a405 100644 --- a/burpui/misc/backend/interface.py +++ b/burpui/misc/backend/interface.py @@ -99,15 +99,17 @@ class BUIbackend(with_metaclass(ABCMeta, object)): conf.update_defaults(self.defaults) section = 'Burp' if section not in conf.options: - section = 'Burp{}'.format(self._vers) - if section in conf.options: + section_old = 'Burp{}'.format(self._vers) + if section_old in conf.options: # TODO: remove the compatibility self.logger.critical( 'The "[{}]" section is DEPRECATED and will be removed ' 'in v0.7.0. Please use the "[Burp]" section ' - 'instead.'.format(section) + 'instead.'.format(section_old) ) + section = section_old conf.default_section(section) + self.with_celery = conf.get('WITH_CELERY', False) self.port = conf.safe_get('bport', 'integer') self.host = conf.safe_get('bhost') self.burpbin = self._get_binary_path( @@ -251,7 +253,7 @@ class BUIbackend(with_metaclass(ABCMeta, object)): [ "client1\t2\ti\t576 0 1443766803", - "client2\t2\ti\t1 0 1422189120", + "client2\t2\ti\t1 0 1422189120" ] """ raise NotImplementedError("Sorry, the current Backend does not implement this method!") # pragma: no cover @@ -484,34 +486,34 @@ class BUIbackend(with_metaclass(ABCMeta, object)): "last": "2015-01-25 13:32:00", "name": "client2", "state": "idle" - }, + } ] """ raise NotImplementedError("Sorry, the current Backend does not implement this method!") # pragma: no cover -# @abstractmethod -# def get_client_status(self, client=None, agent=None): -# """The :func:`burpui.misc.backend.interface.BUIbackend.get_client_status` -# function returns the status of a given client with its last stats. -# -# :param client: What client status do we want -# :type client: str -# :param agent: What server to ask (only in multi-agent mode) -# :type agent: str -# -# :returns: The last status of a given client -# -# Example:: -# -# { -# "name": "client1", -# "state": "idle", -# "percent": null, -# "phase": null, -# } -# -# """ -# raise NotImplementedError("Sorry, the current Backend does not implement this method!") # pragma: no cover + @abstractmethod + def get_client_status(self, name=None, agent=None): + """The :func:`burpui.misc.backend.interface.BUIbackend.get_client_status` + function returns the status of a given client with its last stats. + + :param name: What client status do we want + :type name: str + :param agent: What server to ask (only in multi-agent mode) + :type agent: str + + :returns: The last status of a given client + + Example:: + + { + "state": "idle", + "percent": null, + "phase": null, + "last": "never" + } + + """ + raise NotImplementedError("Sorry, the current Backend does not implement this method!") # pragma: no cover @abstractmethod def get_client(self, name=None, agent=None): @@ -536,7 +538,7 @@ class BUIbackend(with_metaclass(ABCMeta, object)): "encrypted": true, "number": "1", "received": 889818873, - "size": 35612321050, + "size": 35612321050 } ] """ @@ -577,7 +579,7 @@ class BUIbackend(with_metaclass(ABCMeta, object)): "encrypted": true, "number": "1", "received": 889818873, - "size": 35612321050, + "size": 35612321050 } ] """ @@ -767,8 +769,8 @@ class BUIbackend(with_metaclass(ABCMeta, object)): "Mon,Tue,Thu,Fri,17,18,19,20,21,22,23", "Wed,Sat,Sun,06,07,08,09,10,11,12,13,14,15,16,17,18,19,20,21,22,23" ] - }, - ], + } + ] } """ raise NotImplementedError("Sorry, the current Backend does not implement this method!") # pragma: no cover @@ -806,7 +808,7 @@ class BUIbackend(with_metaclass(ABCMeta, object)): raise NotImplementedError("Sorry, the current Backend does not implement this method!") # pragma: no cover @abstractmethod - def store_conf_cli(self, data, client=None, conf=None, agent=None): + def store_conf_cli(self, data, client=None, conf=None, template=False, agent=None): """The :func:`burpui.misc.backend.interface.BUIbackend.store_conf_cli` function works the same way as the :func:`burpui.misc.backend.interface.BUIbackend.store_conf_srv` function diff --git a/burpui/misc/backend/multi.py b/burpui/misc/backend/multi.py index 16976556..199d2ce2 100644 --- a/burpui/misc/backend/multi.py +++ b/burpui/misc/backend/multi.py @@ -373,7 +373,7 @@ class NClient(BUIbackend): self.ssl = ssl self.app = app self.timeout = timeout or 5 - self.version = None + self._agent_version = None def __getattribute__(self, name): # always return this value because we need it and if we don't do that @@ -398,15 +398,15 @@ class NClient(BUIbackend): return object.__getattribute__(self, name) def _get_agent_version(self): - if self.ping() and not self.version: + if self.ping() and not self._agent_version: data = {'func': 'agent_version'} try: - self.version = json.loads(self.do_command(data)) + self._agent_version = json.loads(self.do_command(data)) except BUIserverException: # just ignore the error if this custom function is not # implemented pass - return self.version + return self._agent_version def ping(self): """Check if we are connected to the agent""" @@ -511,13 +511,13 @@ class NClient(BUIbackend): """ @implement - def store_conf_cli(self, data, client=None, conf=None, agent=None): + def store_conf_cli(self, data, client=None, conf=None, template=False, agent=None): """See :func:`burpui.misc.backend.interface.BUIbackend.store_conf_cli`""" # serialize data as it is a nested dict import hmac import hashlib from base64 import b64encode - if not isinstance(data, _ImmutableMultiDict): + if not isinstance(data, (_ImmutableMultiDict, ImmutableMultiDict)): msg = 'Wrong data type' self.logger.warning(msg) raise BUIserverException(msg) @@ -527,7 +527,7 @@ class NClient(BUIbackend): data = ImmutableMultiDict(data.to_dict(False)) key = '{}{}'.format(self.password, 'store_conf_cli') key = to_bytes(key) - pickles = b64encode(pickle.dumps({'data': data, 'conf': conf, 'client': client}, 2)) + pickles = b64encode(pickle.dumps({'data': data, 'conf': conf, 'client': client, 'template': template}, 2)) bytes_pickles = to_bytes(pickles) digest = hmac.new(key, bytes_pickles, hashlib.sha1).hexdigest() data = {'func': 'store_conf_cli', 'args': pickles, 'pickled': True, 'digest': digest} @@ -540,7 +540,7 @@ class NClient(BUIbackend): import hmac import hashlib from base64 import b64encode - if not isinstance(data, _ImmutableMultiDict): + if not isinstance(data, (_ImmutableMultiDict, ImmutableMultiDict)): msg = 'Wrong data type' self.logger.warning(msg) raise BUIserverException(msg) diff --git a/burpui/misc/parser/burp1.py b/burpui/misc/parser/burp1.py index 77775e3d..cf80b22f 100644 --- a/burpui/misc/parser/burp1.py +++ b/burpui/misc/parser/burp1.py @@ -41,8 +41,12 @@ class Parser(Doc): self._server_conf = {} self._client_conf = {} self._clients_conf = {} + self._templates_conf = {} self.clientconfdir = None self.clientconfdir_mtime = None + self.templates = [] + self.templates_dir = '.buitemplates' + self.templates_path = None self.filescache = {} self._configs = {} self.root = None @@ -89,6 +93,7 @@ class Parser(Doc): self._server_conf.clear() self._client_conf.clear() self._clients_conf.clear() + self._list_templates(True) self._list_clients(True) def _load_conf_srv(self): @@ -96,6 +101,12 @@ class Parser(Doc): self._server_conf = Config(self.conf, self, 'srv') self._server_conf.parse() self.clientconfdir = self._server_conf.get('clientconfdir') + self.templates_path = os.path.join(self.clientconfdir, self.templates_dir) + if not os.path.exists(self.templates_path): + try: + os.makedirs(self.templates_path, 0o755) + except OSError as exp: + self.logger.warning(str(exp)) def _load_conf_cli(self): """Load the client configuration file""" @@ -120,17 +131,39 @@ class Parser(Doc): conf.parse() self._clients_conf[cli['name']] = conf + def _load_conf_templates(self, name=None, in_path=None): + """Load all templates configuration""" + if name: + templates = [{'name': name, 'value': in_path}] + else: + templates = self._list_templates(True) + + for template in templates: + conf = self.server_conf.clone() + path = os.path.join(self.templates_path, template['name']) + if template['name'] not in self._templates_conf: + conf.add_file(path) + conf.set_default(path) + conf.parse() + self._templates_conf[template['name']] = conf + def _load_all_conf(self): """Load all configurations""" self._load_conf_srv() self._load_conf_cli() self._load_conf_clients() + self._load_conf_templates() def _new_client_conf(self, name, path): """Create new client conf""" self._load_conf_clients(name, path) return self.clients_conf[name] + def _new_template_conf(self, name, path): + """Create new template conf""" + self._load_conf_templates(name, path) + return self._templates_conf[name] + def _clientconfdir_changed(self): """Detect changes in clientconfdir""" if not self.clientconfdir: @@ -153,6 +186,17 @@ class Parser(Doc): self._clients_conf[name].parse() return self._clients_conf[name] + def _get_template(self, name, path=None): + """Return template conf and refresh it if necessary""" + if self._clientconfdir_changed() and name not in self._templates_conf: + self._templates_conf.clear() + self._load_conf_templates() + if name not in self._templates_conf: + return self._new_template_conf(name, path) + if self._templates_conf[name].changed: + self._templates_conf[name].parse() + return self._templates_conf[name] + def _get_config(self, path, mode='cli'): """Return conf by it's path""" if path in self._configs: @@ -171,7 +215,7 @@ class Parser(Doc): path = os.path.normpath(path) cond = [path.startswith(x) for x in self.backend.includes] - if not any(cond): + if not any(cond) and self.backend.enforce: self.logger.warning( 'Tried to access non-allowed path: {}'.format(path) ) @@ -200,6 +244,27 @@ class Parser(Doc): self.clientconfdir_mtime = os.path.getmtime(self.clientconfdir) return res + def _list_templates(self, force=False): + res = [] + if not self.clientconfdir or not os.path.isdir(self.templates_path): + return res + + if self.templates and not force and not self._clientconfdir_changed(): + return self.templates + + for tpl in os.listdir(self.templates_path): + full_file = os.path.join(self.templates_path, tpl) + if (os.path.isfile(full_file) and not tpl.startswith('.') and + not tpl.endswith('~')): + res.append({ + 'name': tpl, + 'value': os.path.join(self.templates_dir, tpl) + }) + + self.templates = res + self.clientconfdir_mtime = os.path.getmtime(self.clientconfdir) + return res + def _get_server_path(self, name=None, fil=None): """Returns the path of the 'server *fil*' file""" if not name: @@ -256,7 +321,7 @@ class Parser(Doc): return False return self.openssl_auth.check_client_revoked(client) - def remove_client(self, client=None, keepconf=False, delcert=False, revoke=False): + def remove_client(self, client=None, keepconf=False, delcert=False, revoke=False, template=False): """See :func:`burpui.misc.parser.interface.BUIparser.remove_client`""" res = [] revoked = False @@ -265,12 +330,15 @@ class Parser(Doc): return [[NOTIF_ERROR, "No client provided"]] try: if not keepconf: - path = os.path.join(self.clientconfdir, client) + if template: + path = os.path.join(self.templates_path, client) + else: + path = os.path.join(self.clientconfdir, client) os.unlink(path) res.append([NOTIF_OK, "'{}' successfully removed".format(client)]) removed = True - if client in self._clients_conf: + if client in self._clients_conf and not template: del self._clients_conf[client] self._refresh_cache() @@ -302,7 +370,7 @@ class Parser(Doc): return res - def read_client_conf(self, client=None, conf=None): + def read_client_conf(self, client=None, conf=None, template=False): """ See :func:`burpui.misc.parser.interface.BUIparser.read_client_conf` """ @@ -313,7 +381,9 @@ class Parser(Doc): u'multi': [], u'includes': [], u'includes_ext': [], - u'clients': self._list_clients(), + u'templates': [], + u'hierarchy': [], + u'raw': None, } if not client and not conf: return res @@ -322,8 +392,12 @@ class Parser(Doc): if not mconf: if not self.clientconfdir: return res - mconf = os.path.join(self.clientconfdir, client) - config = self._get_client(client, mconf) + if template: + mconf = os.path.join(self.templates_path, client) + config = self._get_template(client, mconf) + else: + mconf = os.path.join(self.clientconfdir, client) + config = self._get_client(client, mconf) else: config = self._get_config(mconf) @@ -336,11 +410,14 @@ class Parser(Doc): res2[u'boolean'] = parsed.boolean res2[u'integer'] = parsed.integer res2[u'multi'] = parsed.multi + res2[u'templates'] = parsed.template res2[u'includes'] = [ x for x in parsed.flatten('include', False).keys() ] res2[u'includes_ext'] = parsed.include + res2[u'hierarchy'] = config.tree + res2[u'raw'] = str(parsed) res.update(res2) self.filescache[mconf] = { @@ -359,9 +436,11 @@ class Parser(Doc): u'boolean': [], u'integer': [], u'multi': [], + u'pair': [], u'includes': [], u'includes_ext': [], - u'clients': self._list_clients(), + u'hierarchy': [], + u'raw': None, } if not conf: mconf = self.conf @@ -374,22 +453,19 @@ class Parser(Doc): if mconf in self.filescache and self.filescache[mconf]['md5'] == parsed.md5: return self.filescache[mconf]['dict'] - clientconfdir = parsed.get('clientconfdir') - if clientconfdir and clientconfdir.parse() != self.clientconfdir: - self.clientconfdir = clientconfdir.parse() - self.clientconfdir_mtime = -1 - res['clients'] = self._list_clients() - res2 = {} res2[u'common'] = parsed.string res2[u'boolean'] = parsed.boolean res2[u'integer'] = parsed.integer res2[u'multi'] = parsed.multi + res2[u'pair'] = parsed.pair res2[u'includes'] = [ x for x in parsed.flatten('include', False).keys() ] res2[u'includes_ext'] = parsed.include + res2[u'hierarchy'] = self.server_conf.tree + res2[u'raw'] = str(parsed) res.update(res2) self.filescache[mconf] = { @@ -401,34 +477,43 @@ class Parser(Doc): def list_clients(self): """See :func:`burpui.misc.parser.interface.BUIparser.list_clients`""" self.read_server_conf() - if not self.clientconfdir: - return [] - return self._list_clients() - def store_client_conf(self, data, client=None, conf=None): + def list_templates(self): + """See :func:`burpui.misc.parser.interface.BUIparser.list_templates`""" + self.read_server_conf() + return self._list_templates() + + def store_client_conf(self, data, client=None, conf=None, template=False): """ See :func:`burpui.misc.parser.interface.BUIparser.store_client_conf` """ if conf and not os.path.isabs(conf): conf = os.path.join(self.clientconfdir, conf) if not conf and not client: + if template: + return [[NOTIF_ERROR, 'Sorry, no template defined']] return [[NOTIF_ERROR, 'Sorry, no client defined']] elif client and not conf: - conf = os.path.join(self.clientconfdir, client) - ret = self.store_conf(data, conf, client, mode='cli') + if template: + if not self.templates_path: + return [[NOTIF_ERROR, 'Sorry, no template directory found']] + conf = os.path.join(self.templates_path, client) + else: + conf = os.path.join(self.clientconfdir, client) + ret = self.store_conf(data, conf, client, mode='cli', template=template) self._refresh_cache() # refresh client list return ret def store_conf(self, data, conf=None, client=None, mode='srv', - insecure=False): + insecure=False, template=False): """See :func:`burpui.misc.parser.interface.BUIparser.store_conf`""" mconf = None if not conf: mconf = self.conf else: mconf = conf - if mconf != self.conf and not mconf.startswith('/'): + if mconf != self.conf and not os.path.isabs(mconf): mconf = os.path.join(self.root, mconf) if not mconf: return [[NOTIF_WARN, 'Sorry, no configuration file defined']] @@ -443,7 +528,9 @@ class Parser(Doc): ] check = False - if client: + if template: + conffile = self._get_template(client, mconf).get_file(mconf) + elif client: conffile = self._get_client(client, mconf).get_file(mconf) else: conffile = self.server_conf.get_file(mconf) @@ -459,6 +546,53 @@ class Parser(Doc): return ret + def remove_conf(self, path=None): + """See :func:`burpui.misc.parser.interface.BUIparser.remove_conf`""" + if not path: + return [ + [ + NOTIF_WARN, + 'No file selected for removal' + ] + ] + if path == self.conf: + return [ + [ + NOTIF_ERROR, + 'Removing the burp-server configuration file is not supported' + ] + ] + + parsed = self.server_conf.get_file(self.conf) + includes = parsed.include + if includes: + for include in includes: + if 'value' in include and path in include['value']: + try: + os.unlink(path) + return [ + [ + NOTIF_OK, + "File '{}' successfully removed".format(path) + ] + ] + except IOError as exp: + return [ + [ + NOTIF_ERROR, + "Unable to remove configuration file '{}': {}".format( + path, + str(exp) + ) + ] + ] + return [ + [ + NOTIF_ERROR, + "No file suited for removal" + ] + ] + def cancel_restore(self, name=None): """See :func:`burpui.misc.parser.interface.BUIparser.cancel_restore`""" path = self._get_server_restore_path(name) diff --git a/burpui/misc/parser/burp2.py b/burpui/misc/parser/burp2.py index d67ee365..c81a0d0a 100644 --- a/burpui/misc/parser/burp2.py +++ b/burpui/misc/parser/burp2.py @@ -18,16 +18,38 @@ class Parser(Burp1): """Extends :class:`burpui.misc.parser.burp1.Parser`""" pver = 2 + pair_srv = [ + u'port', + u'max_children', + u'status_port', + u'max_status_children', + ] + pair_associations = { + u'port': u'max_children', + u'max_children': u'port', + u'status_port': u'max_status_children', + u'max_status_children': u'status_port', + } + integer_srv = Burp1.integer_srv + for rem in ['port', 'max_children', 'status_port', 'max_status_children']: + integer_srv.remove(rem) + advanced_type = Burp1.advanced_type + advanced_type.update({ + u'port': u'integer', + u'max_children': u'integer', + u'status_port': u'integer', + u'max_status_children': u'integer', + }) multi_srv = Burp1.multi_srv + [ u'label', ] string_srv = Burp1.string_srv + [ u'manual_delete', + u'rblk_memory_max', ] boolean_add = [ u'acl', u'xattr', - u'server_can_override_includes', u'glob_after_script_pre', u'cname_fqdn', u'cname_lowercase', @@ -55,17 +77,20 @@ class Parser(Burp1): u'randomise': __(u"max secs"), u'manual_delete': __(u"path"), u'label': __(u"some informations"), - u'server_can_override_includes': u"0|1", + u'status_address': __(u"address|localhost"), u'glob_after_script_pre': u"0|1", u'enabled': u"0|1", u'cname_fqdn': u"0|1", u'cname_lowercase': u"0|1", + u'rblk_memory_max': u"b/Kb/Mb/Gb", }) + values = Burp1.values + # status_address can now listen on any address + del values['status_address'] defaults = Burp1.defaults defaults.update({ u'acl': True, u'xattr': True, - u'server_can_override_includes': True, u'glob_after_script_pre': True, u'randomise': 0, u'manual_delete': u'', @@ -73,6 +98,7 @@ class Parser(Burp1): u'enabled': True, u'cname_fqdn': True, u'cname_lowercase': False, + u'rblk_memory_max': u'256Mb', }) doc = Burp1.doc doc.update({ @@ -105,10 +131,11 @@ class Parser(Burp1): " output. The idea is to provide a mechanism for" " arbitrary values to be passed to clients of the server" " status monitor."), - u'server_can_override_includes': __(u"To prevent the server from being" - " able to override your local" - " include/exclude list, set this" - " to 0. The default is 1."), + u'status_address': __(u"Defines the main TCP address that the server " + "listens on for status requests. The default " + "is special value 'localhost' that includes " + "both '::1' (if available) and '127.0.0.1' " + "(always)."), u'glob_after_script_pre': __(u"Set this to 0 if you do not want" " include_glob settings to be evaluated" " after the pre script is run. The" @@ -132,4 +159,32 @@ class Parser(Burp1): " The default is 0. When set to 1 the name" " provided by the client while authenticating" " will be lowercased."), + u'port': __(u"Defines the main TCP port that the server listens on. " + "Specify multiple 'port' entries on separate lines in " + "order to listen on multiple ports. Each port can be " + "configured with its own 'max_children' value."), + u'max_children': __(u"Defines the number of child processes to fork " + "(the number of clients that can simultaneously " + "connect. The default is 5. Specify multiple " + "'max_children' entries on separate lines if you " + "have configured multiple port entries."), + u'status_port': __(u"Defines the TCP port that the server listens on " + "for status requests. Comment this out to have no " + "status server. Specify multiple 'status_port' " + "entries on separate lines in order to listen on " + "multiple ports. Each port can be configured with " + "its own 'max_status_children' value."), + u'max_status_children': __(u"Defines the number of status child " + "processes to fork (the number of status " + "clients that can simultaneously connect. " + "The default is 5. Specify multiple " + "'max_status_children' entries on separate " + "lines if you have configured multiple " + "status_port entries."), + u'rblk_memory_max': __("The maximum amount of data from the disk " + "cached in server memory during a protocol2 " + "restore/verify. The default is 256Mb. This " + "option can be overriden per-client in the " + "client configuration files in clientconfdir " + "on the server."), }) diff --git a/burpui/misc/parser/doc.py b/burpui/misc/parser/doc.py index cb42d2b4..d2da2938 100644 --- a/burpui/misc/parser/doc.py +++ b/burpui/misc/parser/doc.py @@ -123,6 +123,7 @@ class Doc(BUIparser): } placeholders = { u'.': __(u"path or glob"), + u'address': __(u"address"), u'atime': u"0|1", u'autoupgrade_dir': __(u"path"), u'ca_burp_ca': __(u"path"), @@ -240,6 +241,11 @@ class Doc(BUIparser): u'ssl_key', u'timer_script', ] + advanced_type = { + u'keep': u'integer', + } + pair_srv = [] + pair_associations = {} multi_srv = [ u'exclude_comp', u'exclude_ext', diff --git a/burpui/misc/parser/interface.py b/burpui/misc/parser/interface.py index 0f650465..322e2149 100644 --- a/burpui/misc/parser/interface.py +++ b/burpui/misc/parser/interface.py @@ -103,6 +103,30 @@ class BUIparser(with_metaclass(ABCMeta, object)): ] }, ], + "hierarchy": [ + { + "children": [ + { + "children": [], + "dir": "/tmp/burp/conf.d", + "full": "/tmp/burp/conf.d/empty.conf", + "name": "empty.conf", + "parent": "/tmp/burp/burp-server.conf" + }, + { + "children": [], + "dir": "/tmp/burp/conf.d", + "full": "/tmp/burp/conf.d/ipv4.conf", + "name": "ipv4.conf", + "parent": "/tmp/burp/burp-server.conf" + } + ], + "dir": "/tmp/burp", + "full": "/tmp/burp/burp-server.conf", + "name": "burp-server.conf", + "parent": null + } + ] } """ raise NotImplementedError( @@ -110,7 +134,7 @@ class BUIparser(with_metaclass(ABCMeta, object)): ) # pragma: no cover @abstractmethod - def store_client_conf(self, data, client=None, conf=None): + def store_client_conf(self, data, client=None, conf=None, template=False): """:func:`burpui.misc.parser.interface.BUIparser.store_client_conf` is used by :func:`burpui.misc.backend.BUIbackend.store_conf_cli`. @@ -120,6 +144,12 @@ class BUIparser(with_metaclass(ABCMeta, object)): :param client: Name of the client for which to apply this config :type client: str + + :param conf: The explicit filename of the conf + :type conf: str + + :param template: Is this file a template + :type template: bool """ raise NotImplementedError( "Sorry, the current Parser does not implement this method!" @@ -127,7 +157,7 @@ class BUIparser(with_metaclass(ABCMeta, object)): @abstractmethod def store_conf(self, data, conf=None, client=None, mode='srv', - insecure=False): + insecure=False, template=False): """:func:`burpui.misc.parser.interface.BUIparser.store_conf` is used to store the configuration from the web-ui into the actual configuration files. @@ -149,6 +179,9 @@ class BUIparser(with_metaclass(ABCMeta, object)): :param insecure: Used for the CLI :type insecure: bool + :param template: Is it a template + :type template: bool + :returns: A list of notifications to return to the UI (success or failure) @@ -160,6 +193,19 @@ class BUIparser(with_metaclass(ABCMeta, object)): "Sorry, the current Parser does not implement this method!" ) # pragma: no cover + @abstractmethod + def remove_conf(self, path=None): + """:func:`burpui.misc.parser.interface.BUIparser.remove_conf` + is used to remove a configuration file. + It only works for "included" files within the server configuration file. + + :param path: The path of the file to remove + :type path: str + """ + raise NotImplementedError( + "Sorry, the current Parser does not implement this method!" + ) # pragma: no cover + @abstractmethod def path_expander(self, pattern=None, source=None, client=None): """:func:`burpui.misc.parser.interface.BUIparser.path_expander` is used @@ -192,6 +238,17 @@ class BUIparser(with_metaclass(ABCMeta, object)): "Sorry, the current Parser does not implement this method!" ) # pragma: no cover + @abstractmethod + def list_templates(self): + """:func:`burpui.misc.parser.interface.BUIparser.list_templates` is used + to retrieve a list of templates with their absolute paths. + + :returns: A list of templates + """ + raise NotImplementedError( + "Sorry, the current Parser does not implement this method!" + ) # pragma: no cover + @abstractmethod def is_client_revoked(self, client=None): """:func:`burpui.misc.parser.interface.BUIparser.is_client_revoked` is @@ -207,7 +264,7 @@ class BUIparser(with_metaclass(ABCMeta, object)): ) # pragma: no cover @abstractmethod - def remove_client(self, client=None, keepconf=False, delcert=False, revoke=False): + def remove_client(self, client=None, keepconf=False, delcert=False, revoke=False, template=False): """:func:`burpui.misc.parser.interface.BUIparser.remove_client` is used to delete a client from burp's configuration. @@ -223,6 +280,9 @@ class BUIparser(with_metaclass(ABCMeta, object)): :param revoke: Whether to revoke the associated certificate :type revoke: bool + :param template: Whether we remove a template + :type template: bool + :returns: A list of notifications to return to the UI (success or failure) """ @@ -231,7 +291,7 @@ class BUIparser(with_metaclass(ABCMeta, object)): ) # pragma: no cover @abstractmethod - def read_client_conf(self, client=None, conf=None): + def read_client_conf(self, client=None, conf=None, template=False): """:func:`burpui.misc.parser.interface.BUIparser.read_client_conf` is called by :func:`burpui.misc.backend.interface.BUIbackend.read_conf_cli` in order to parse the burp-clients configuration files. diff --git a/burpui/misc/parser/utils.py b/burpui/misc/parser/utils.py index 1d948435..ac218d50 100644 --- a/burpui/misc/parser/utils.py +++ b/burpui/misc/parser/utils.py @@ -23,6 +23,8 @@ from ...datastructures import MultiDict RESET_IDENTIFIER = '_reset_bui_CUSTOM' +BEGIN_TEMPLATES = 'BURP-UI TEMPLATES' +END_TEMPLATES = 'END TEMPLATES' class Option(object): @@ -103,6 +105,11 @@ class Option(object): """Option to string""" return self.dump() + def __eq__(self, other): + other_name = getattr(other, 'name', None) + other_value = getattr(other, 'value', None) + return self.name == other_name and self.value == other_value + class OptionStr(Option): """Option type String @@ -123,7 +130,10 @@ class OptionInt(Option): def parse(self): """Parse the option value""" - return int(self.value) + try: + return int(self.value) + except (ValueError, TypeError): + return 0 class OptionBool(Option): @@ -170,74 +180,6 @@ class OptionBool(Option): return self._parse(self.value) -class OptionMulti(Option): - """Option type Multi - - Example: - keep = 7 - keep = 4 - """ - type = 'multi' - - def __init__(self, name, value=None): - self.name = name - self._dirty = True - self._is_reset = [] - if value: - if not isinstance(value, list): - self.value = [value] - else: - self.value = value - else: - self.value = [] - self.idx = len(self.value) - 1 - - def append(self, value, reset=None): - self._dirty = True - self.value.append(value) - if reset is not None: - self.set_reset(reset) - return self.value - - def remove(self, value): - self._dirty = True - self.value.remove(value) - return self.value - - def index(self, value): - return self.value.index(value) - - def len(self): - return len(self.value) - - def dump_init(self, start=0): - """Initialize the dump""" - self.dump_idx = start - - def dump(self, start=0): - """Return the option representation to store in configuration file""" - self.dump_init(start) - ret = u'' - if start > len(self.value): - return ret - for idx in range(start, len(self.value)): - ret += '{}\n'.format(self.dump_index(idx)) - return ret.rstrip('\n') - - def dump_index(self, index): - if index > len(self.value): - index = len(self.value) - 1 - val = self.value[index] - delim = self.delim - self.idx = index - if self.is_reset(): - delim = self.reset_delim - return '{} {} {}'.format(self.name, delim, val) - - def get_reset(self): - return self._is_reset - - class OptionInc(Option): """Option type Include @@ -272,7 +214,7 @@ class OptionInc(Option): def _path_absolute(self, path): absolute = path - if not path.startswith('/'): + if not os.path.isabs(path): if self.root: absolute = os.path.join(self.root, path) elif self.mode == 'srv': @@ -321,6 +263,333 @@ class OptionInc(Option): return '' +class OptionTpl(Option): + """Option type Template + + Example: + . .buitemplates/windows + """ + type = 'template' + delim = "" + + def __init__(self, parser, name, value=None): + """ + :param parser: Parser instance + :type parser: :class:`burpui.misc.parser.burp1.Parser` + """ + super(OptionTpl, self).__init__(name, value) + self.parser = parser + self.extended = False + self._dirty = True + if name: + self._id = name.split(os.path.sep)[-1] + else: + self._id = '' + + @property + def dirty(self): + return self._dirty + + def _path_absolute(self, path): + absolute = path + if not os.path.isabs(path): + absolute = os.path.join(self.parser.clientconfdir, path) + return absolute + + def extend(self): + """Helper function for the parsing""" + if not self._dirty and self.extended: + return self.extended + path = self._path_absolute(self.value) + self.clean() + self.extended = path + return path + + def parse(self): + """Parse the option value""" + return self.extend() + + def dump(self): + """Return the option representation to store in configuration file""" + if self.extend() and not self.parser.backend.enforce: + return '. {}'.format(self.name) + # if the include did not match anything, we can safely remove it + return '' + + +option_for_type = { + 'integer': OptionInt, + 'string': OptionStr, + 'boolean': OptionBool, + 'include': OptionInc, +} + + +class OptionMulti(Option): + """Option type Multi + + Example: + keep = 7 + keep = 4 + """ + type = 'multi' + + def __init__(self, parser, name, value=None): + self.parser = parser + self.name = name + self._dirty = True + self._is_reset = [] + self.content_type = self.parser.advanced_type.get(name, 'string') + self.associate = getattr(self.parser, 'pair_associations', {}).get(self.name) + self._init_value(value) + self.idx = len(self.value) - 1 + + def _init_value(self, value): + if value: + container = self._obj_for_type() + if not isinstance(value, list): + value = container(self.name, value) + self.value = [value] + else: + self.value = [container(self.name, x) for x in value] + else: + self.value = [] + + def _obj_for_type(self): + return option_for_type.get(self.content_type, OptionStr) + + def _wrap_object(self, value): + container = self._obj_for_type() + return container(self.name, value) + + def append(self, value, reset=None): + self._dirty = True + if isinstance(value, list): + for v in value: + self.value.append(self._wrap_object(v)) + else: + self.value.append(self._wrap_object(value)) + if reset is not None: + self.set_reset(reset) + return self.value + + def update(self, value): + """Change the option value""" + self._dirty = True + self._init_value(value) + self._is_reset = [] + + def remove(self, value): + idx = self.value(value) + del self.value[idx] + self._dirty = True + return self.value + + def index(self, value): + for i, x in enumerate(self): + if x == value: + return i + raise ValueError('{} is not in list'.format(value)) + + def len(self): + return len(self.value) + + def dump(self, start=0, strict=True): + """Return the option representation to store in configuration file""" + ret = u'' + if start > len(self.value): + return ret + res = [self.dump_index(i, strict) for i in range(start, len(self.value))] + ret = u'\n'.join(res) + return ret.rstrip('\n') + + def dump_index(self, index, strict=True): + if index >= len(self.value): + return '' + val = self.value[index] + delim = self.delim + self.idx = index + if self.is_reset(): + delim = self.reset_delim + return '{} {} {}'.format(self.name, delim, sanitize_string('{}'.format(val.parse()), strict)) + + def parse(self): + return [x.parse() for x in self.value] + + def get_reset(self): + return self._is_reset + + def __len__(self): + return len(self.value) + + def __getitem__(self, ii): + return self.value[ii].parse() + + def __delitem__(self, ii): + del self.value[ii] + self._dirty = True + + def __setitem__(self, ii, val): + self.value[ii] = self._wrap_object(val) + self._dirty = True + + def insert(self, ii, val): + self.value.insert(ii, self._wrap_object(val)) + + def __iter__(self): + for x in self.value: + yield x.parse() + + +class OptionPair(Option, dict): + """Option type Pair + + Example: + port = 4971 + max_children = 5 + """ + type = 'pair' + + def __init__(self, parser, name, value=None): + self.parser = parser + self.name = name + self.association = getattr(self.parser, 'pair_associations', {}).get(self.name) + self._dirty = True + self._init_value(name, value) + # is there a special := syntax for this option + self._is_reset = [] + + def _init_value(self, name, value): + self.value = {} + if value: + value = OptionMulti(self.parser, name, value) + self.value[name] = value + + def append(self, name, value): + self._dirty = True + if name != self.name: + self.association = name + if name in self.value: + self.value[name].append(value) + else: + self.value[name] = OptionMulti(self.parser, name, value) + return self.value + + def update(self, name, value): + """Change the option value""" + self._dirty = True + self._init_value(name, value) + self._is_reset = [] + + def remove(self, name, value): + self._dirty = True + try: + self.value.get(name, []).remove(value) + except ValueError: + pass + return self.value + + def index(self, name, value): + try: + return self.value.get(name, []).index(value) + except ValueError: + return -1 + + def len(self, name): + if name in self.value: + return self.value[name].len() + return -1 + + def dump(self, start=0, strict=True): + """Return the option representation to store in configuration file""" + ret = u'' + try: + self_length = len(self.value[self.name]) + except KeyError: + self_length = -1 + try: + associate_length = len(self.value[self.association]) + except KeyError: + associate_length = -1 + if start >= self_length and start >= associate_length: + return ret + for idx in range(start, max(self_length, associate_length)): + v1 = self.dump_index(self.name, idx, strict) + v2 = self.dump_index(self.association, idx, strict) + if v1: + ret += '{}\n'.format(v1) + if v2: + ret += '{}\n'.format(v2) + return ret.rstrip('\n') + + def dump_index(self, name, index, strict): + length = self.len(name) + try: + length = len(self.value[name]) + except KeyError: + length = -1 + if index >= length: + return u'' + return self.value.get(name).dump_index(index, strict) + + def parse(self, key=None): + if key: + if key not in self.value: + return [] + return self.value[key].parse() + ret = {} + for key, opts in iteritems(self.value): + ret[key] = opts.parse() + return ret + + def __setitem__(self, key, item): + self.append(key, item) + + def __getitem__(self, key): + return self.value.get(key, OptionMulti(self.parser, key)) + + def get(self, key, default=None): + try: + return self.value[key] + except KeyError: + if default: + return default + return OptionMulti(self.parser, key) + + def __len__(self): + return len(self.value) + + def __delitem__(self, key): + del self.value[key] + + def clear(self): + self.value.clear() + + def copy(self): + return self.value.copy() + + def has_key(self, key): + return key in self.value + + def keys(self): + return self.value.keys() + + def values(self): + return self.value.values() + + def items(self): + return self.value.items() + + def pop(self, *args): + return self.value.pop(*args) + + def __contains__(self, item): + return item in self.value + + def __iter__(self): + return iter(self.value) + + class File(dict): """Object representing a configuration file @@ -345,6 +614,8 @@ class File(dict): self._dirty = False # _changed is used to know if the file changed since last read self._changed = True + # _parsing_templates is used to know if we are currently parsing templates + self._parsing_templates = False # cache the content of the file self._raw = [] self._raw_data = MultiDict() @@ -354,6 +625,7 @@ class File(dict): self.name = name self.parent = parent self.updated = [] + self.associations = set() self.reset = {} self.options = OrderedDict() self.types = { @@ -361,7 +633,9 @@ class File(dict): 'integer': OrderedDict(), 'include': OrderedDict(), 'multi': OrderedDict(), + 'pair': OrderedDict(), 'string': OrderedDict(), + 'template': OrderedDict(), } if self.name: self.parse() @@ -372,6 +646,10 @@ class File(dict): if val.dirty: self._changed = True return self._changed + for key, val in iteritems(self.types['template']): + if val.dirty: + self._changed = True + return self._changed try: if self.name: mtime = os.path.getmtime(self.name) @@ -398,6 +676,8 @@ class File(dict): for key, val in iteritems(self.options): if isinstance(val, OptionMulti) and not raw: ret.setlist(key, val.parse()) + elif isinstance(val, OptionPair) and not raw: + ret.setlist(key, val.parse(key)) else: ret[key] = val if raw else val.parse() return ret @@ -445,6 +725,13 @@ class File(dict): ret[key] = opt.parse() if parse else opt return ret + def flatten_obj(self, name, obj, parse=True): + return { + 'name': name, + 'value': obj.parse() if parse else obj, + 'reset': obj.get_reset() + } + @property def dirty(self): if not self._dirty: @@ -459,10 +746,24 @@ class File(dict): def integer(self): return self.flatten('integer') + @property + def pair(self): + return self.flatten('pair') + @property def include(self): return self.flatten('include') + @property + def template(self): + ret = [] + for tpl in self.flatten('template', parse=False): + ret.append({ + 'value': tpl['name'], + 'name': tpl['value']._id, + }) + return ret + @property def multi(self): return self.flatten('multi') @@ -477,7 +778,8 @@ class File(dict): self.types[key] = OrderedDict() for key, opt in iteritems(self.options): - self.types[opt.type][key] = opt + if key not in self.associations: + self.types[opt.type][key] = opt self._dirty = False @@ -488,7 +790,7 @@ class File(dict): if opt == u'.': return 'include' - for typ in ['boolean', 'integer', 'multi', 'string']: + for typ in ['boolean', 'integer', 'multi', 'string', 'pair']: if opt in self._options_for_type(typ): return typ return None @@ -501,12 +803,14 @@ class File(dict): if typ == 'integer': return OptionInt(key, value) if typ == 'multi': - opt = self.options.get(key, OptionMulti(key)) + opt = self.options.get(key, OptionMulti(self.parser, key)) opt.append(value) return opt + if typ == 'pair': + return OptionPair(self.parser, key, value) if typ == 'include': key = value - opt = OptionInc( + return OptionInc( self.parser, key, value, @@ -530,11 +834,15 @@ class File(dict): def get(self, key, default=None): try: + if key in self._options_for_type('pair'): + return self.options[key].get(key) return self.options[key] except KeyError: return default def __getitem__(self, key): + if key in self._options_for_type('pair'): + return self.options[key].get(key) return self.options[key] def __setitem__(self, key, value): @@ -544,34 +852,52 @@ class File(dict): elif key in self._options_for_type('integer'): opt = OptionInt(key, value) elif key in self._options_for_type('multi'): - opt = self.options.get(key, OptionMulti(key)) + opt = self.options.get(key, OptionMulti(self.parser, key)) opt.append(value) + elif key in self._options_for_type('pair'): + association = self.parser.pair_associations.get(key) + if key not in self.options and association not in self.options: + self.associations.add(association) + opt = OptionPair(self.parser, key) + elif association in self.options: + opt = self.options.get(association) + else: + opt = self.options.get(key) + opt.append(key, value) elif key == u'.': key = value - opt = OptionInc( - self.parser, - key, - value, - root=self.name, - mode=self.mode - ) + if self._parsing_templates: + opt = OptionTpl(self.parser, key, value) + else: + opt = OptionInc( + self.parser, + key, + value, + root=self.name, + mode=self.mode + ) else: opt = OptionStr(key, value) self.options[key] = opt - self.types[opt.type][key] = opt + if key not in self.associations: + self.types[opt.type][key] = opt def __repr__(self): self._refresh_types() ret = u'' for key, opts in iteritems(self.types): ret += '{} =>\n'.format(key) - for _, opt in iteritems(opts): + for key2, opt in iteritems(opts): + if key2 in self.associations: + continue ret += '\t' + repr(opt) + '\n' return ret.rstrip('\n') def __str__(self): ret = u'' - for _, val in iteritems(self.options): + for key, val in iteritems(self.options): + if key in self.associations: + continue tmp = str(val) if tmp: ret += tmp + '\n' @@ -651,6 +977,7 @@ class File(dict): return ret def _write_key(self, fil, key, data, index=None, dry=False): + strict = 'regex' not in key if not dry: self._changed = True @@ -659,17 +986,38 @@ class File(dict): val = sanitize_string(data) fil.write('. {}\n'.format(val)) return val - # don't need to parse data again if index > 0 - if index is not None and index > 0: - fil.write('{}\n'.format(self[key].dump_index(index))) + + if index is not None and index >= 0: + if key not in self.updated: + val = data.getlist(key) + if key in self: + self[key].update(val) + else: + self[key] = val + self.updated.append(key) + if key in self.reset: + self[key].set_resets(self.reset[key]) + fil.write('{}\n'.format(self[key].dump_index(index, strict))) + return None + + if key in getattr(self.parser, 'multi_{}'.format(self.mode)) or \ + key in getattr(self.parser, 'pair_{}'.format(self.mode), []): + if key not in self.updated: + val = data.getlist(key) + if key in self: + self[key].update(val) + else: + self[key] = val + self.updated.append(key) + if key in self.reset: + self[key].set_resets(self.reset[key]) + val = self[key] + fil.write('{}\n'.format(self[key].dump(strict=strict))) return None - strict = 'regex' not in key if key not in self.updated: if key in self.parser.boolean_srv or key in self.parser.boolean_cli: val = data.get(key) - elif key in self.parser.multi_srv or key in self.parser.multi_cli: - val = [sanitize_string(x, strict) for x in data.getlist(key)] else: val = sanitize_string(str(data.get(key)), strict) if key in self: @@ -679,16 +1027,13 @@ class File(dict): self.updated.append(key) if key in self.reset: self[key].set_resets(self.reset[key]) + if dry: ret = self[key].parse() if index is not None: return ret[index] return ret - if index is not None: - self[key].dump_init() - fil.write('{}\n'.format(self[key].dump_index(index))) - else: - fil.write('{}\n'.format(self[key])) + fil.write('{}\n'.format(self[key])) def parse(self, force=False): """Parse the current config""" @@ -698,6 +1043,10 @@ class File(dict): self.clear() for line in self.raw: if re.match(r'^\s*#', line): + if BEGIN_TEMPLATES in line: + self._parsing_templates = True + if END_TEMPLATES in line: + self._parsing_templates = False continue res = re.search(r'\s*([^=\s]+)\s*(:)?=?\s*(.*)$', line) if res: @@ -710,7 +1059,10 @@ class File(dict): val = val.replace('gzip', 'zlib') self[key] = val if key in self: - self[key].set_reset(reset is not None) + try: + self[key].set_reset(reset is not None) + except AttributeError: + pass self._dirty = False @@ -724,7 +1076,7 @@ class File(dict): filename = os.path.basename(dest) if dirname and not os.path.exists(dirname): try: - os.makedirs(dirname) + os.makedirs(dirname, 0o755) except OSError as exp: return [[NOTIF_WARN, str(exp)]] @@ -784,7 +1136,9 @@ class File(dict): newkeys = list(set(viewkeys(data)) - set(oldkeys)) multi_index_map = {} - already_multi = [] + pair_index_map = {} + already_multi = set() + already_pair = set() already_file = [] written = [] self.updated = [] @@ -821,7 +1175,21 @@ class File(dict): with codecs.open(dest, 'w', 'utf-8', errors='ignore') as fil: # f.write('# Auto-generated configuration using Burp-UI\n') data_keys = list(data.keys()) + if len(self.template) > 0 or 'templates' in data: + _dump(' {}'.format(BEGIN_TEMPLATES), True) + tpls = data.getlist('templates') or [x['value'] for x in self.template] + for tpl in tpls: + self._write_key(fil, '.', tpl) + _dump(' {}'.format(END_TEMPLATES), True) + skip_line = False for idx, line in enumerate(orig): + if self._line_is_comment(line) and BEGIN_TEMPLATES in line: + skip_line = True + if self._line_is_comment(line) and END_TEMPLATES in line: + skip_line = False + continue + if skip_line: + continue key = self._get_line_key(line, False) if (self._line_removed(line, data_keys) and not self._line_is_comment(line) and @@ -848,16 +1216,20 @@ class File(dict): # The line is still present or has been un-commented, # rewrite it with eventual changes multi = key in getattr(self.parser, 'multi_{}'.format(self.mode)) + pair = key in getattr(self.parser, 'pair_{}'.format(self.mode), []) + if pair and key not in pair_index_map: + pair_index_map[key] = 0 if multi and key not in multi_index_map: multi_index_map[key] = 0 if key in written: _dump(line, comment=(not self._line_is_comment(line))) else: if multi: + length = len(self[key]) if key in self else -1 if key not in already_multi and \ (key not in self or (key in self and - self[key].len() > multi_index_map[key])): + length > multi_index_map[key])): self._write_key( fil, key, @@ -868,9 +1240,8 @@ class File(dict): else: _dump(line, comment=(not self._line_is_comment(line))) continue - if self[key].len() == multi_index_map[key]: - if key not in already_multi: - already_multi.append(key) + if len(self[key]) == multi_index_map[key]: + already_multi.add(key) continue # dump the rest of the multi if there are no # more keys in the conf @@ -878,9 +1249,35 @@ class File(dict): rest = self[key].dump(multi_index_map[key]) if rest: fil.write('{}\n'.format(rest)) - multi_index_map[key] = self[key].len() - if key not in already_multi: - already_multi.append(key) + multi_index_map[key] = length + already_multi.add(key) + elif pair: + length = len(self[key]) if key in self else -1 + if key not in already_pair and \ + (key not in self or + (key in self and + length > pair_index_map[key])): + self._write_key( + fil, + key, + data, + pair_index_map[key] + ) + pair_index_map[key] += 1 + else: + _dump(line, comment=(not self._line_is_comment(line))) + continue + if len(self[key]) == pair_index_map[key]: + already_pair.add(key) + continue + # dump the rest of the pair if there are no more + # keys in the conf + if not _is_key_after(key, idx + 1): + rest = self[key].dump(pair_index_map[key]) + if rest: + fil.write('{}\n'.format(rest)) + pair_index_map[key] = length + already_pair.add(key) else: # The line was a comment and there was a further # matching setting, so we just jump to the @@ -912,26 +1309,31 @@ class File(dict): else: _dump(line, comment=(key in written and not self._line_is_comment(line))) - # Write the new keys - for key in newkeys: - if key.endswith(RESET_IDENTIFIER): - continue - if (key not in written and key not in already_multi and - key not in ['includes', 'includes_ori']): - self._write_key( - fil, - key, - data, - ) # write the rest of the multi settings for key, idx in iteritems(multi_index_map): if key not in already_multi and idx < self[key].len(): fil.write('{}\n'.format(self[key].dump(idx))) + # write the rest of the pair settings + for key, idx in iteritems(pair_index_map): + if key not in already_pair and idx < self[key].len(): + fil.wrrite('{}\n'.format(self[key].dump(idx))) # Write the rest of file inclusions if 'includes' in data: for inc in data.getlist('includes'): if inc not in already_file: self._write_key(fil, '.', inc) + # Write the new keys + for key in newkeys: + if key.endswith(RESET_IDENTIFIER): + continue + if key not in written and key not in already_multi and \ + key not in already_pair and \ + key not in ['includes', 'includes_ori', 'templates']: + self._write_key( + fil, + key, + data, + ) except Exception as exp: return [[NOTIF_ERROR, str(exp)]] @@ -1016,6 +1418,7 @@ class Config(File): self.default = path self.name = path self._includes = [] + self._templates = [] self._dirty = True if path: self.files[path] = File(parser, path, mode=mode) @@ -1039,6 +1442,11 @@ class Config(File): path = os.path.join(os.path.dirname(root), path) self.add_file(path, root) self._includes.append(path) + for key, path in iteritems(conf.flatten('template', False)): + if not os.path.isabs(path): + path = os.path.join(os.path.dirname(root), path) + self.add_file(path, root) + self._templates.append(path) # recursively parse the conf if orig != self.files: @@ -1049,13 +1457,14 @@ class Config(File): return del self._includes[:] + del self._templates[:] self._parse() removed = [] orig = self.files for path, conf in iteritems(orig): - if conf.parent and (conf.name not in self._includes or - conf.name in removed): + if conf.parent and ((conf.name not in self._includes and + conf.name not in self._templates) or conf.name in removed): removed.append(path) self.del_file(path) @@ -1072,6 +1481,7 @@ class Config(File): basename = os.path.basename(name) return { 'name': basename, + 'title': basename, 'full': name, 'dir': dirname, 'parent': parent, @@ -1092,6 +1502,8 @@ class Config(File): for idx, (top, conf) in enumerate(iteritems(self.files)): if idx < offset: continue + if idx > offset: + break node = __new_node(conf.name) for key, val in iteritems(conf.flatten('include', False)): for path in val: @@ -1107,7 +1519,7 @@ class Config(File): if conf and conf in self.files: return self.files[conf].store(dest, insecure) for name, conf in iteritems(self.files): - ret += conf.store(dest, insecure) + ret += conf.store(insecure=insecure) return ret def store_data(self, conf, data, insecure=False): @@ -1148,7 +1560,7 @@ class Config(File): return self.files[idx] def get_file(self, path): - ret = self.files.get(path, File(self.parser, mode=self.mode)) + ret = self.files.get(path, File(self.parser, path, mode=self.mode)) ret.parse() return ret @@ -1175,23 +1587,30 @@ class Config(File): # now update caches with new values for _, fil in iteritems(self.files): self.options.update(fil.options) + self.associations = self.associations.union(fil.associations) # FIXME: find a way to cache efficiently # fil.clean() for key, val in iteritems(self.options): - self.types[val.type][key] = val + if key not in self.associations: + self.types[val.type][key] = val self._dirty = False def _get(self, key, default=None, raw=False): self._refresh() try: - obj = self.options[key] + if key in self._options_for_type('pair'): + obj = self.options[key].get(key) + else: + obj = self.options[key] except KeyError: + if default: + return default if self.parser and key in self.parser.defaults: obj = self._new_opt(key, self.parser.defaults[key]) else: - return default + return None return obj if raw else obj.parse() def get_raw(self, key, default=None): @@ -1211,6 +1630,8 @@ class Config(File): def __getitem__(self, key): self._refresh() + if key in self._options_for_type('pair'): + return self.options[key].get(key) return self.options[key] def __setitem__(self, key, value): diff --git a/burpui/plugins.py b/burpui/plugins.py index 744b60fb..389e0f50 100644 --- a/burpui/plugins.py +++ b/burpui/plugins.py @@ -25,6 +25,7 @@ class PluginManager(object): self.app = app self.searchpath = searchpath self.init = False + self.loaded = False self.plugins = {} def _init_manager(self): @@ -34,8 +35,11 @@ class PluginManager(object): self.plugin_source = self.plugin_base.make_plugin_source( searchpath=self.searchpath ) + self.init = True - def load_all(self): + def load_all(self, force=False): + if self.loaded and not force: + return self._init_manager() for plugin_name in self.plugin_source.list_plugins(): if plugin_name not in self.plugins: @@ -63,6 +67,7 @@ class PluginManager(object): str(exp) ) ) + self.loaded = True def get_plugins_by_type(self, plugin_type): ret = {} diff --git a/burpui/routes.py b/burpui/routes.py index 7b08b5c5..59258927 100644 --- a/burpui/routes.py +++ b/burpui/routes.py @@ -7,6 +7,7 @@ .. moduleauthor:: Ziirish """ +import re import math import uuid @@ -62,6 +63,12 @@ def bytes_human(value): return '{0:.1eM}'.format(_hr(value)) +@view.app_template_filter() +def regex_replace(value, regex, replace): + """Replace every string matching the given regex with the replacement""" + return re.sub(regex, replace, value, flags=re.IGNORECASE) + + """ And here is the main site """ @@ -101,7 +108,8 @@ def calendar(server=None, client=None): @login_required def settings(server=None, conf=None): # Only the admin can edit the configuration - if not current_user.is_anonymous and not current_user.acl.is_admin(): + if not current_user.is_anonymous and not current_user.acl.is_admin() and \ + not current_user.acl.is_moderator(): abort(403) if not conf: try: @@ -124,7 +132,8 @@ def settings(server=None, conf=None): @login_required def admin(): # Only the admin can access this page - if not current_user.is_anonymous and not current_user.acl.is_admin(): + if not current_user.is_anonymous and not current_user.acl.is_admin() and \ + not current_user.acl.is_moderator(): abort(403) return render_template('admin.html', admin=True, ng_controller='AdminCtrl') @@ -144,7 +153,8 @@ def me(): @login_required def cli_settings(server=None, client=None, conf=None): # Only the admin can edit the configuration - if not current_user.is_anonymous and not current_user.acl.is_admin(): + if not current_user.is_anonymous and not current_user.acl.is_admin() and \ + not current_user.acl.is_moderator(): abort(403) if not conf: try: @@ -162,9 +172,11 @@ def cli_settings(server=None, client=None, conf=None): pass client = client or request.args.get('client') server = server or request.args.get('serverName') + template = request.args.get('template') or False return render_template( 'settings.html', settings=True, + template=template, client=client, server=server, conf=conf, @@ -290,15 +302,15 @@ def client_report(server=None, name=None): """Specific client report""" server = server or request.args.get('serverName') try: - l = bui.client.get_client(name, agent=server) + res = bui.client.get_client(name, agent=server) except BUIserverException: - l = [] - if len(l) == 1: + res = [] + if len(res) == 1: return redirect( url_for( '.backup_report', name=name, - backup=l[0]['number'], + backup=res[0]['number'], server=server ) ) diff --git a/burpui/security.py b/burpui/security.py index d4525afc..de5ab476 100644 --- a/burpui/security.py +++ b/burpui/security.py @@ -26,7 +26,7 @@ def sanitize_string(string, strict=True, paranoid=False): else: import re ret = repr(string).replace('\\\\', '\\') - ret = re.sub(r"^u?'(.*)'$", r"\1", ret) + ret = re.sub(r"^u?(?P['\"])(.*)(?P=quote)$", r"\1", ret) return to_unicode(ret) diff --git a/burpui/server.py b/burpui/server.py index 09334d87..04a7df5b 100644 --- a/burpui/server.py +++ b/burpui/server.py @@ -8,6 +8,7 @@ """ import os +import re import sys import logging import traceback @@ -26,6 +27,8 @@ G_PORT = 5000 G_BIND = u'::' G_REFRESH = 180 G_LIVEREFRESH = 5 +G_IGNORE_LABELS = [] +G_FORMAT_LABELS = [] G_SSL = False G_SINGLE = True G_SSLCERT = u'' @@ -42,6 +45,7 @@ G_RATIO = u'60/minute' G_CELERY = False G_SCOOKIE = True G_DEMO = False +G_DSN = u'' G_APPSECRET = u'random' G_COOKIETIME = 14 G_SESSIONTIME = 5 @@ -49,6 +53,7 @@ G_DATABASE = u'' G_PREFIX = u'' G_PLUGINS = [] G_NO_SERVER_RESTORE = False +G_WS_ENABLED = True G_WS_EMBEDDED = False G_WS_BROKER = u'redis' G_WS_URL = u'' @@ -76,10 +81,13 @@ class BUIServer(Flask): 'prefix': G_PREFIX, 'plugins': G_PLUGINS, 'demo': G_DEMO, + 'dsn': G_DSN, }, 'UI': { 'refresh': G_REFRESH, 'liverefresh': G_LIVEREFRESH, + 'ignore_labels': G_IGNORE_LABELS, + 'format_labels': G_FORMAT_LABELS, }, 'Security': { 'scookie': G_SCOOKIE, @@ -98,6 +106,7 @@ class BUIServer(Flask): 'ratio': G_RATIO, }, 'WebSocket': { + 'enabled': G_WS_ENABLED, 'embedded': G_WS_EMBEDDED, 'broker': G_WS_BROKER, 'url': G_WS_URL, @@ -169,6 +178,7 @@ class BUIServer(Flask): 'demo', 'boolean' ) + self.config['BUI_DSN'] = self.conf.safe_get('dsn') self.bind = self.config['BUI_BIND'] = self.conf.safe_get('bind') version = self.conf.safe_get('version', 'integer') if unittest and version != 1: @@ -182,7 +192,7 @@ class BUIServer(Flask): key = 'standalone' if 'standalone' in \ self.conf.conf.get(self.conf.section, {}) else 'single' if key == 'standalone': - # TODO: remove the conpatibility in v0.7.0 + # TODO: remove the compatibility in v0.7.0 self.logger.warning( 'The "standalone" option is DEPRECATED and has been replaced ' 'by the "single" option. Please update your conf before we ' @@ -229,6 +239,21 @@ class BUIServer(Flask): 'integer', 'UI' ) + self.ignore_labels = self.conf.safe_get( + 'ignore_labels', + 'force_list', + 'UI' + ) + format_labels = self.conf.safe_get( + 'format_labels', + 'force_list', + 'UI' + ) + self.format_labels = [] + for format_label in format_labels: + search = re.search(r'^s(?P.)(?P.*?)(?P=separator)(?P.*?)(?P=separator)$', format_label) + if search: + self.format_labels.append((search.group('regex'), search.group('replace'))) # Production options self.storage = self.config['BUI_STORAGE'] = self.conf.safe_get( @@ -282,6 +307,11 @@ class BUIServer(Flask): self.use_celery.lower() != 'none' # WebSocket options + self.ws_enabled = self.config['WS_ENABLED'] = self.conf.safe_get( + 'enabled', + 'boolean', + section='WebSocket' + ) self.websocket = self.config['WITH_WS'] = self.conf.safe_get( 'embedded', 'boolean', diff --git a/burpui/static/dashboard.css b/burpui/static/dashboard.css index b93561a8..303ac8cb 100644 --- a/burpui/static/dashboard.css +++ b/burpui/static/dashboard.css @@ -184,7 +184,7 @@ width:100%; text-transform: uppercase; } -#tree { +#tree, #tree-hierarchy { background-color: #E9F2F9; color: #697075; } @@ -372,3 +372,47 @@ td { stroke: #555; stroke-width: 1px; } + +#back-to-top { + position: fixed; + bottom: 10px; + left: 10px; + background: #3e444c; + width: 50px; + height: 50px; + display: block; + text-decoration: none; + -webkit-border-radius: 35px; + -moz-border-radius: 35px; + border-radius: 35px; + display: none; + -webkit-transition: all 0.3s linear; + -moz-transition: all 0.3s ease; + -ms-transition: all 0.3s ease; + -o-transition: all 0.3s ease; + transition: all 0.3s ease; +} +#back-to-top span { + color: #fff; + margin: 0; + position: relative; + left: 16px; + top: 13px; + font-size: 19px; + -webkit-transition: all 0.3s ease; + -moz-transition: all 0.3s ease; + -ms-transition: all 0.3s ease; + -o-transition: all 0.3s ease; + transition: all 0.3s ease; +} +#back-to-top:hover { + background: #428bca; +} +#back-to-top:hover span { + color: #fff; + top: 5px; +} + +a { + outline: 0; +} diff --git a/burpui/static/vendor b/burpui/static/vendor index f8bb1797..ccc31f1e 160000 --- a/burpui/static/vendor +++ b/burpui/static/vendor @@ -1 +1 @@ -Subproject commit f8bb1797b28cbb005ac40c52990fbf713db1ad2e +Subproject commit ccc31f1e0fabddb9759cfe1d1ae3c36f370b9610 diff --git a/burpui/tasks.py b/burpui/tasks.py index 73a008a7..37ae0f7d 100644 --- a/burpui/tasks.py +++ b/burpui/tasks.py @@ -55,9 +55,9 @@ BEAT_SCHEDULE = { 'task': '{}.ping_backend'.format(ME), 'schedule': crontab(minute='15'), # run every hour }, - 'backup-running-minutely': { + 'backup-running-4-minutely': { 'task': '{}.backup_running'.format(ME), - 'schedule': timedelta(seconds=30), # run every 30 seconds + 'schedule': timedelta(seconds=15), # run every 15 seconds }, 'get-all-backups-every-twenty-minutes': { 'task': '{}.get_all_backups'.format(ME), @@ -129,15 +129,15 @@ def wait_for(lock_name, value, wait=10, timeout=LOCK_EXPIRE): return old_lock -@celery.task +@celery.task(ignore_result=True) def ping_backend(): if bui.standalone: - bui.client.status() + bui.client.status(sync=True) else: def __status(server): (serv, back) = server try: - return bui.client.status(agent=serv) + return bui.client.status(sync=True, agent=serv) except BUIserverException: return False @@ -147,7 +147,7 @@ def ping_backend(): )) -@celery.task(bind=True) +@celery.task(bind=True, ignore_result=True) def backup_running(self): # run once at the time, if one task was already running, we just discard # the new attempt @@ -174,7 +174,7 @@ def backup_running(self): release_lock(self.name) -@celery.task(bind=True) +@celery.task(bind=True, ignore_result=True) def get_all_backups(self): # run once at the time, if one task was already running, we just discard # the new attempt @@ -195,7 +195,7 @@ def get_all_backups(self): release_lock(self.name) -@celery.task(bind=True) +@celery.task(bind=True, ignore_result=True) def get_all_clients_reports(self): # run once at the time, if one task was already running, we just discard # the new attempt @@ -213,7 +213,7 @@ def get_all_clients_reports(self): release_lock(self.name) -@celery.task +@celery.task(ignore_result=True) def cleanup_expired_sessions(): def expires(sess): ret = session_manager.invalidate_session_by_id(sess.uuid) @@ -223,7 +223,7 @@ def cleanup_expired_sessions(): list(map(expires, session_manager.get_expired_sessions())) -@celery.task +@celery.task(ignore_result=True) def cleanup_restore(): tasks = db.session.query(Task).filter(Task.task == 'perform_restore').filter(datetime.utcnow() > Task.expire).all() # tasks = Task.query.filter_by(task='perform_restore').all() @@ -263,7 +263,7 @@ def cleanup_restore(): @celery.task(bind=True) def perform_restore(self, client, backup, files, strip, fmt, passwd, server=None, user=None, - expire=timedelta(minutes=60)): + admin=False, room=None, expire=timedelta(minutes=60)): ret = None lock_name = '{}-{}'.format(self.name, server) @@ -305,7 +305,8 @@ def perform_restore(self, client, backup, 'filename': filename, 'path': archive, 'user': user, - 'server': server + 'server': server, + 'admin': admin } logger.debug(ret) diff --git a/burpui/templates/500_sentry.html b/burpui/templates/500_sentry.html new file mode 100644 index 00000000..556b12fb --- /dev/null +++ b/burpui/templates/500_sentry.html @@ -0,0 +1,11 @@ + + + +{% if event_id %} + +{% endif %} diff --git a/burpui/templates/admin.html b/burpui/templates/admin.html index 6d8d109a..cdb05bcd 100644 --- a/burpui/templates/admin.html +++ b/burpui/templates/admin.html @@ -4,11 +4,33 @@
{% include "small_topbar.html" %}
-

nyan

-
+
+ {{ _('Users') }} +
+ + + + + + + + + + + + +
{{ _('Username') }}{{ _('Authentication Backend') }}{{ _('Role') }}{{ _('Groups') }}{{ _('Controls') }}
+
{% endblock %} diff --git a/burpui/templates/backup-report.html b/burpui/templates/backup-report.html index 8c343f61..67464d35 100644 --- a/burpui/templates/backup-report.html +++ b/burpui/templates/backup-report.html @@ -19,40 +19,52 @@

{{ _('Backup n°%(number)s of %(client)s', number=mypad(nbackup), client=cname) }}

-
- +
+

{{ _('New') }}

+
+ +
-

{{ _('New') }}

-
- +
+

{{ _('Changed') }}

+
+ +
-

{{ _('Changed') }}

-
- +
+

{{ _('Unchanged') }}

+
+ +
-

{{ _('Unchanged') }}

-
- +
+

{{ _('Deleted') }}

+
+ +
-

{{ _('Deleted') }}

-
- +
+

{{ _('Total') }}

+
+ +
-

{{ _('Total') }}

-
- +
+

{{ _('Scanned') }}

+
+ +
-

{{ _('Scanned') }}

-
- +
+
+ +
diff --git a/burpui/templates/client-browse.html b/burpui/templates/client-browse.html index c58c5a07..aef4c162 100644 --- a/burpui/templates/client-browse.html +++ b/burpui/templates/client-browse.html @@ -10,7 +10,7 @@
  • {{ _('%(client)s overview', client=cname) }}
  • {{ _('Backup n°%(number)s overview', number=mypad(nbackup)) }}
    • {% else -%} -
  • Home
    • +
  • {{ _('Home') }}
  • {{ _('%(client)s overview', client=cname) }}
  • {{ _('Backup n°%(number)s overview', number=mypad(nbackup)) }}
    • {% endif -%} @@ -123,7 +123,7 @@
    -