bui-monitor and async backends fully supported (closes #278)

This commit is contained in:
ziirish 2018-08-17 15:41:32 +02:00
parent 12b8fcc003
commit 97b35373a9
No known key found for this signature in database
GPG key ID: 72DB229A64B54E46
9 changed files with 246 additions and 116 deletions

View file

@ -307,14 +307,21 @@ def compile_translation():
help='Database to connect to for persistent storage')
@click.option('-p', '--plugins', default=None,
help='Plugins location')
@click.option('-m', '--monitor', default=None,
help='bui-monitor configuration file')
@click.option('-C', '--concurrency', default=None, type=click.INT,
help='Number of concurrent async process to spawn')
@click.option('-B', '--backend', default=None,
help='Switch to another backend', type=click.Choice(['burp2', 'async']))
@click.option('-n', '--dry', is_flag=True,
help='Dry mode. Do not edit the files but display changes')
def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry):
def setup_burp(bconfcli, bconfsrv, client, host, redis, database,
plugins, monitor, concurrency, backend, dry):
"""Setup burp client for burp-ui."""
if app.config['BACKEND'] != 'burp2':
if app.config['BACKEND'] not in ['burp2', 'async']:
click.echo(
click.style(
'Sorry, you can only setup the Burp 2 client',
'Sorry, you can only setup the burp2 and the async backends',
fg='red'
),
err=True
@ -331,6 +338,22 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry):
)
sys.exit(1)
if concurrency:
from multiprocessing import cpu_count
if concurrency > cpu_count():
click.echo(
click.style(
'Warning: setting a concurrency level higher than the available CPU'
' count might cause you some troubles',
fg='yellow'
)
)
is_async = app.config['BACKEND'] == 'async' or (backend and backend == 'async')
if backend and app.config['BACKEND'] != backend:
app.config['BACKEND'] = backend
try:
msg = app.load_modules(True)
except Exception as e:
@ -341,9 +364,25 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry):
from .misc.parser.utils import Config
from .app import get_redis_server
from .config import BUIConfig
import difflib
import tempfile
if monitor:
monconf = BUIConfig(monitor)
monconf_orig = []
mon_orig = mon_source = monitor
if dry:
try:
with open(monitor) as fil:
monconf_orig = fil.readlines()
except:
pass
(_, temp) = tempfile.mkstemp()
monconf.options.filename = temp
parser = app.client.get_parser()
orig = source = None
conf_orig = []
@ -376,17 +415,24 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry):
refresh = True
if (database or redis) and not app.conf.lookup_section('Production', source):
refresh = True
if concurrency and not app.conf.lookup_section('Async', source):
refresh = True
if refresh:
app.conf._refresh(True)
def _edit_conf(key, val, attr, section='Burp', obj=app.client):
if monitor and not monconf.lookup_section('Global', mon_source):
monconf._refresh(True)
def _edit_conf(key, val, attr, section='Burp', obj=app.client, conf=app.conf):
if val and (((key not in app.conf.options[section]) or
(key in app.conf.options[section] and
val != app.conf.options[section][key])) and
getattr(obj, attr) != val):
app.conf.options[section][key] = val
app.conf.options.write()
((obj and getattr(obj, attr) != val) or (not obj))):
conf.options[section][key] = val
conf.options.write()
if obj:
setattr(obj, attr, val)
click.echo(
click.style(
'Adding new option: "{}={}" to section [{}]'.format(
@ -413,10 +459,30 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry):
refresh |= _edit_conf('bconfcli', bconfcli, 'burpconfcli')
refresh |= _edit_conf('bconfsrv', bconfsrv, 'burpconfsrv')
refresh |= _edit_conf('plugins', plugins, 'plugins', 'Global', app)
if backend:
refresh |= _edit_conf('backend', backend, None, 'Global', None)
if is_async and concurrency:
refresh |= _edit_conf('concurrency', concurrency, None, 'Async', None)
if refresh:
app.conf._refresh(True)
if monitor and concurrency:
if _edit_conf('pool', concurrency, None, 'Global', None, monconf):
monconf._refresh(True)
if monitor and app.config['BACKEND'] == 'async':
mon_password = monconf.options['Global'].get('password')
back_password = app.conf.options['Async'].get('password')
if mon_password != back_password:
click.echo(
click.style(
'Backend password does not match monitor password',
fg='yellow'
)
)
if redis:
try:
# detect missing modules
@ -513,6 +579,26 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry):
if out:
click.echo_via_pager(out)
if dry and monitor:
temp = monconf.options.filename
monconf.options.filename = mon_orig
after = []
try:
if not os.path.exists(temp) or os.path.getsize(temp) == 0:
after = monconf_orig
else:
with open(temp) as fil:
after = fil.readlines()
os.unlink(temp)
except:
pass
diff = difflib.unified_diff(monconf_orig, after, fromfile=mon_orig, tofile='{}.new'.format(mon_orig))
out = ''
for line in diff:
out += _color_diff(line)
if out:
click.echo_via_pager(out)
bconfcli = bconfcli or app.conf.options['Burp'].get('bconfcli') or \
getattr(app.client, 'burpconfcli')
bconfsrv = bconfsrv or app.conf.options['Burp'].get('bconfsrv') or \
@ -533,6 +619,7 @@ def setup_burp(bconfcli, bconfsrv, client, host, redis, database, plugins, dry):
confsrv.set_default(bconfsrv)
confsrv.parse()
# TODO: support upcoming 'listen' and 'listen_status' options
if host not in ['::1', '127.0.0.1']:
bind = confsrv.get('status_address')
if (bind and bind not in [host, '::', '0.0.0.0']) or not bind:
@ -776,10 +863,10 @@ password = abcdefgh
help='Show you some tips')
def diag(client, host, tips):
"""Check Burp-UI is correctly setup."""
if app.config['BACKEND'] != 'burp2':
if app.config['BACKEND'] not in ['burp2', 'async']:
click.echo(
click.style(
'Sorry, you can only setup the Burp 2 client',
'Sorry, you can only diag the burp2 and the async backends',
fg='red'
),
err=True
@ -1118,7 +1205,7 @@ def sysinfo(verbose, load):
except Exception as e:
msg = str(e)
backend_version = app.config['BACKEND']
backend = app.config['BACKEND']
colors = {
'True': 'green',
@ -1133,7 +1220,7 @@ def sysinfo(verbose, load):
if platform.system() == 'Linux':
click.echo('Distribution: {} {} {}'.format(*platform.dist()))
click.echo('Single mode: {}'.format(app.config['STANDALONE']))
click.echo('Backend version: {}'.format(backend_version))
click.echo('Backend: {}'.format(backend))
click.echo('WebSocket embedded: {}'.format(click.style(embedded_ws, fg=colors[embedded_ws])))
click.echo('WebSocket available: {}'.format(click.style(available_ws, colors[available_ws])))
click.echo('Config file: {}'.format(app.config.conffile))

View file

@ -1,4 +1,4 @@
FROM python:3.6-alpine3.7
FROM python:3.7-alpine3.8
RUN apk add --no-cache supervisor logrotate librsync libressl tzdata nginx \
&& apk add --no-cache --virtual .fetch-deps \
@ -46,8 +46,6 @@ RUN apk add --no-cache supervisor logrotate librsync libressl tzdata nginx \
)" \
&& apk add --virtual .python-rundeps $runDeps \
&& apk del .build-deps \
# needed for the bui-cli tool
&& ln -sf /usr/bin/nc /bin/nc \
# do some cleanup
&& rm -rf /usr/src/burp /usr/src/uthash ~/.cache

View file

@ -74,7 +74,7 @@ backlog = 2048
#
workers = @GUNICORN_WORKERS@ * 2
worker_class = 'gevent'
worker_class = '@GUNICORN_WORKER_CLASS@'
worker_connections = 1000
timeout = 300
keepalive = 2
@ -187,6 +187,7 @@ proc_name = None
# A callable that takes a server instance as the sole argument.
#
def post_fork(server, worker):
from psycogreen.gevent import patch_psycopg
patch_psycopg()
if worker_class == 'gevent':
def post_fork(server, worker):
from psycogreen.gevent import patch_psycopg
patch_psycopg()

View file

@ -4,6 +4,26 @@ set -e
SETUP_DIR="/app/setup"
CONFIG_DIR="${SETUP_DIR}/config"
BURPUI_CONFIG=${BURPUI_CONFIG:-/etc/burp/burpui.cfg}
BURPUI_MONITOR_CONFIG=${BURPUI_MONITOR_CONFIG:-/etc/burp/bui-monitor.cfg}
BURPUI_CLIENT_NAME=${BURPUI_CLIENT_NAME:-bui}
BURPUI_VERBOSE=${BURPUI_VERBOSE:-0}
BURPUI_UID=${BURPUI_UID:-5337}
BURPUI_GID=${BURPUI_GID:-5337}
BURPUI_PLUGINS=${BURPUI_PLUGINS:-none}
BURPUI_WS_WORKERS=${BURPUI_WS_WORKERS:-$(getconf _NPROCESSORS_ONLN)}
BURPUI_RP_SCHEME=${BURPUI_RP_SCHEME:-https}
BURPUI_BACKEND=${BURPUI_BACKEND:-async}
BURP_CLIENT_CONFIG=${BURP_CLIENT_CONFIG:-/tmp/burp.conf}
BURP_SERVER_CONFIG=${BURP_SERVER_CONFIG:-/etc/burp/burp-server.conf}
BURP_SERVER_ADDR=${BURP_SERVER_ADDR:-burp-server}
REDIS_SERVER=${REDIS_SERVER:-redis:6379}
DATABASE_URL=${DATABASE_URL:-postgresql://burpui:burpui@pgsql/burpuidb}
GUNICORN_WORKERS=${GUNICORN_WORKERS:-$(getconf _NPROCESSORS_ONLN)}
GUNICORN_WORKER_CLASS=${GUNICORN_WORKER_CLASS:-sync}
ASYNC_POOL_SIZE=${ASYNC_POOL_SIZE:-$(getconf _NPROCESSORS_ONLN)}
TIMEZONE=${TIMEZONE:-Europe/Paris}
trap appStop SIGINT SIGTERM
doas() {
@ -14,22 +34,6 @@ doas() {
appStart () {
BURPUI_CONFIG=${BURPUI_CONFIG:-/etc/burp/burpui.cfg}
BURPUI_CLIENT_NAME=${BURPUI_CLIENT_NAME:-bui}
BURPUI_VERBOSE=${BURPUI_VERBOSE:-0}
BURPUI_UID=${BURPUI_UID:-5337}
BURPUI_GID=${BURPUI_GID:-5337}
BURPUI_PLUGINS=${BURPUI_PLUGINS:-none}
BURPUI_WS_WORKERS=${BURPUI_WS_WORKERS:-$(getconf _NPROCESSORS_ONLN)}
BURPUI_RP_SCHEME=${BURPUI_RP_SCHEME:-https}
BURP_CLIENT_CONFIG=${BURP_CLIENT_CONFIG:-/tmp/burp.conf}
BURP_SERVER_CONFIG=${BURP_SERVER_CONFIG:-/etc/burp/burp-server.conf}
BURP_SERVER_ADDR=${BURP_SERVER_ADDR:-burp-server}
REDIS_SERVER=${REDIS_SERVER:-redis:6379}
DATABASE_URL=${DATABASE_URL:-postgresql://burpui:burpui@pgsql/burpuidb}
GUNICORN_WORKERS=${GUNICORN_WORKERS:-$(getconf _NPROCESSORS_ONLN)}
TIMEZONE=${TIMEZONE:-Europe/Paris}
[ -e /usr/share/zoneinfo/$TIMEZONE ] && {
cp /usr/share/zoneinfo/$TIMEZONE /etc/localtime
echo "$TIMEZONE" >/etc/timezone
@ -54,17 +58,25 @@ appStart () {
chown burpui: $BURPUI_CONFIG
}
[ -e "$BURPUI_MONITOR_CONFIG" ] || {
cp /usr/local/share/burpui/etc/buimonitor.sample.cfg $BURPUI_MONITOR_CONFIG
chown burpui: $BURPUI_MONITOR_CONFIG
}
# wait for redis to be up
sleep 3
LOGFILE=$(doas burpui mktemp)
echo "Setting up burp & burp-ui:"
echo "bui-manage -c $BURPUI_CONFIG setup_burp -b $BURP_CLIENT_CONFIG -s $BURP_SERVER_CONFIG -h $BURP_SERVER_ADDR -c $BURPUI_CLIENT_NAME -r $REDIS_SERVER -d $DATABASE_URL -p $BURPUI_PLUGINS"
doas burpui "bui-manage -c $BURPUI_CONFIG setup_burp -b $BURP_CLIENT_CONFIG -s $BURP_SERVER_CONFIG -h $BURP_SERVER_ADDR -c $BURPUI_CLIENT_NAME -r $REDIS_SERVER -d $DATABASE_URL -p $BURPUI_PLUGINS 2>&1 | tee $LOGFILE"
COMMAND="bui-manage -c $BURPUI_CONFIG setup_burp --burp-conf-cli $BURP_CLIENT_CONFIG --burp-conf-serv $BURP_SERVER_CONFIG --host $BURP_SERVER_ADDR --client $BURPUI_CLIENT_NAME --redis $REDIS_SERVER --database $DATABASE_URL --plugins $BURPUI_PLUGINS --monitor $BURPUI_MONITOR_CONFIG --concurrency $ASYNC_POOL_SIZE --backend $BURPUI_BACKEND"
echo $COMMAND
doas burpui "$COMMAND 2>&1 | tee $LOGFILE"
ret=$?
ASYNC="True"
CELERY="True"
grep -q "Unable to contact the redis server" $LOGFILE && CELERY=""
[ "$GUNICORN_WORKER_CLASS" != "sync" ] || [ "$BURPUI_BACKEND" != "async" ] && ASYNC=""
[ "$ret" != "0" ] && {
echo
@ -111,12 +123,12 @@ EOF
}
# You can change log verbosity at runtime
sed -r "s'@BURPUI_CONFIG@'$BURPUI_CONFIG'" /etc/supervisor.d/gunicorn.ini.sample >/etc/supervisor.d/gunicorn.ini
sed -i -r "s'@BURPUI_VERBOSE@'$BURPUI_VERBOSE'" /etc/supervisor.d/gunicorn.ini
sed -r "s'@BURPUI_CONFIG@'$BURPUI_CONFIG';s'@BURPUI_VERBOSE@'$BURPUI_VERBOSE'" /etc/supervisor.d/gunicorn.ini.sample >/etc/supervisor.d/gunicorn.ini
sed -r "s'@BURPUI_CONFIG@'$BURPUI_CONFIG'" /etc/supervisor.d/bui-celery.ini.sample >/etc/supervisor.d/bui-celery.ini
sed -r "s'@BURPUI_MONITOR_CONFIG@'$BURPUI_MONITOR_CONFIG'" /etc/supervisor.d/bui-monitor.ini.sample >/etc/supervisor.d/bui-monitor.ini
sed -r "s'@GUNICORN_WORKERS@'$GUNICORN_WORKERS'" /etc/burp-ui/burpui_gunicorn.py.sample >/etc/burp-ui/burpui_gunicorn.py
sed -r "s'@GUNICORN_WORKERS@'$GUNICORN_WORKERS';s'@GUNICORN_WORKER_CLASS@'$GUNICORN_WORKER_CLASS'" /etc/burp-ui/burpui_gunicorn.py.sample >/etc/burp-ui/burpui_gunicorn.py
sed -r "s'@HTTP_SCHEME@'$BURPUI_RP_SCHEME'" ${CONFIG_DIR}/nginx/nginx.conf >/etc/nginx/nginx.conf
@ -148,6 +160,11 @@ EOF
supervisorctl start bui-celery >/dev/null
}
[ -n "$ASYNC" ] && {
echo "Starting bui-monitor..."
supervisorctl start bui-monitor >/dev/null
}
echo "Starting gunicorn..."
supervisorctl start gunicorn >/dev/null
@ -173,13 +190,19 @@ appStop() {
echo ""
echo "Stopping nginx..."
supervisorctl stop nginx >/dev/null
[ "$WEBSOCKET" == "True" ] && {
for i in $(seq $BURPUI_WS_WORKERS)
do
echo "Stopping websocket worker $i..."
supervisorctl stop websocket-$i
done
}
echo "Stopping bui-celery..."
supervisorctl stop bui-celery &>/dev/null || true
echo "Stopping bui-monitor..."
supervisorctl stop bui-monitor &>/dev/null || true
for i in $(seq $BURPUI_WS_WORKERS)
do
echo "Stopping websocket worker $i..."
supervisorctl stop websocket-$i &>/dev/null || true
done
echo "Stopping gunicorn..."
supervisorctl stop gunicorn >/dev/null
echo "Stopping crond..."

View file

@ -98,6 +98,19 @@ stdout_logfile=/var/log/supervisor/%(program_name)s.log
stderr_logfile=/var/log/supervisor/%(program_name)s.log
EOF
# configure monitor worker template
cat > /etc/supervisor.d/bui-monitor.ini.sample <<EOF
[program:bui-monitor]
priority=20
directory=/tmp
command=/usr/local/bin/bui-monitor -c @BURPUI_MONITOR_CONFIG@
user=burpui
autostart=false
autorestart=true
stdout_logfile=/var/log/supervisor/%(program_name)s.log
stderr_logfile=/var/log/supervisor/%(program_name)s.log
EOF
# configure gunicorn worker template
cat > /etc/supervisor.d/gunicorn.ini.sample <<EOF
[program:gunicorn]

View file

@ -54,6 +54,7 @@ services:
# Additionally, you can override some default settings
# environment:
# - BURPUI_CONFIG=/etc/burp/burpui.cfg
# - BURPUI_MONITOR_CONFIG=/etc/burp/bui-monitor.cfg
# - BURPUI_VERBOSE=0
# - BURPUI_CLIENT_NAME=bui
# - BURPUI_UID=5337
@ -61,10 +62,13 @@ services:
# - BURPUI_PLUGINS=none
# - BURPUI_WS_WORKERS=2
# - BURPUI_RP_SCHEME=https
# - BURPUI_BACKEND=async
# - BURP_CLIENT_CONFIG=/tmp/burp.conf
# - BURP_SERVER_CONFIG=/etc/burp/burp-server.conf
# - DATABASE_URL=postgresql://burpui:burpui@pgsql/burpuidb
# - GUNICORN_WORKERS=2
# - GUNICORN_WORKER_CLASS=sync
# - REDIS_SERVER=redis:6379
# - BURP_SERVER_ADDR=burp-server
# - ASYNC_POOL_SIZE=2
# - TIMEZONE=Europe/Paris

View file

@ -12,7 +12,7 @@ ssl = true
sslcert = /var/lib/burp/ssl/server/ssl_cert-server.pem
# ssl key
sslkey = /var/lib/burp/ssl/server/ssl_cert-server.key
# burp backend to load either 'burp1' or 'burp2'.
# burp backend to load either 'burp1', 'burp2' or 'async'.
# You can also use whatever custom backend you like if it is located in the
# 'plugins' directory and if it implements the right interface.
backend = burp2
@ -42,21 +42,23 @@ revoke = true
# commands on Unix (the InfoZIP utilities) dont support these extensions. »
zip64 = false
## burp backend specific options
#[Burp]
## burp status address (can only be '127.0.0.1' or '::1')
# burp backend specific options
[Burp]
# burp status address (can only be '127.0.0.1' or '::1')
# DEPRECATED
#bhost = 127.0.0.1
## burp status port
# burp status port
# DEPRECATED
#bport = 4972
## burp binary
#burpbin = /usr/sbin/burp
## vss_strip binary
#stripbin = /usr/sbin/vss_strip
## burp client configuration file used for the restoration
#bconfcli = /etc/burp/burp.conf
## burp server configuration file used for the setting page
#bconfsrv = /etc/burp/burp-server.conf
## temporary directory to use for restoration
#tmpdir = /tmp/bui
## how many time to wait for the monitor to answer (in seconds)
#timeout = 15
# burp binary
burpbin = /usr/sbin/burp
# vss_strip binary
stripbin = /usr/sbin/vss_strip
# burp client configuration file used for the restoration
bconfcli = /etc/burp/burp.conf
# burp server configuration file used for the setting page
bconfsrv = /etc/burp/burp-server.conf
# temporary directory to use for restoration
tmpdir = /tmp/bui
# how many time to wait for the monitor to answer (in seconds)
timeout = 15

View file

@ -17,11 +17,11 @@ sslkey = /var/lib/burp/ssl/server/ssl_cert-server.key
# monitor password
password = password123456
## burp backend specific options
#[Burp]
## burp binary
#burpbin = /usr/sbin/burp
## burp client configuration file used for the restoration
#bconfcli = /etc/burp/burp.conf
## how many time to wait for the monitor to answer (in seconds)
#timeout = 15
# burp backend specific options
[Burp]
# burp binary
burpbin = /usr/sbin/burp
# burp client configuration file used for the restoration
bconfcli = /etc/burp/burp.conf
# how many time to wait for the monitor to answer (in seconds)
timeout = 15

View file

@ -162,40 +162,56 @@ zip64 = true
# `server_can_restore = 0`
noserverrestore = false
## burp backend specific options
#[Burp]
## burp status address (can only be '127.0.0.1' or '::1')
# burp backend specific options
[Burp]
# burp status address (can only be '127.0.0.1' or '::1')
# DEPRECATED
#bhost = ::1
## burp status port
# burp status port
# DEPRECATED
#bport = 4972
## burp binary
#burpbin = /usr/sbin/burp
## vss_strip binary
#stripbin = /usr/sbin/vss_strip
## burp client configuration file used for the restoration
#bconfcli = /etc/burp/burp.conf
## burp server configuration file used for the setting page
#bconfsrv = /etc/burp/burp-server.conf
## temporary directory to use for restoration
#tmpdir = /tmp/bui
## how many time to wait for the monitor to answer (in seconds)
#timeout = 15
# burp binary
burpbin = /usr/sbin/burp
# vss_strip binary
stripbin = /usr/sbin/vss_strip
# burp client configuration file used for the restoration
bconfcli = /etc/burp/burp.conf
# burp server configuration file used for the setting page
bconfsrv = /etc/burp/burp-server.conf
# temporary directory to use for restoration
tmpdir = /tmp/bui
# how many time to wait for the monitor to answer (in seconds)
timeout = 15
## async backend specific options
#[Async]
## address of the monitor pool
#host = ::1
## port of the monitor pool
#port = 11111
## how many time to wait for the monitor pool to answer (in seconds)
#timeout = 15
## monitor pool password
#password = password123456
## enable SSL
#ssl = true
## number of operations to process concurrently
## the value should not exceed the pool size you set in the bui-monitor.cfg file
#concurrency = 2
# async backend specific options
[Async]
# address of the monitor pool
host = ::1
# port of the monitor pool
port = 11111
# how many time to wait for the monitor pool to answer (in seconds)
timeout = 15
# monitor pool password
password = password123456
# enable SSL
ssl = true
# number of operations to process concurrently
# the value should not exceed the pool size you set in the bui-monitor.cfg file
concurrency = 2
# Basic audit backend options
[BASIC:AUDIT]
# Backend priority. Higher is first
priority = 100
# debug level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
# the default is the same as your global application level
level = WARNING
# path to a file to log into
logfile = none
# maximum logfile size
max_bytes = 30 * 1024 * 1024
# number of files to keep
rotate = 5
## ldapauth specific options
#[LDAP]
@ -326,20 +342,6 @@ noserverrestore = false
## As a result, user5 will be granted the following rights:
## '{"ro": {"agents": ["*", "agent1"], "www*": ["desk*"]}, "rw": {"clients": ["dev*"], "www*": ["desk1"]}}
## Basic audit backend options
#[BASIC:AUDIT]
## Backend priority. Higher is first
#priority = 100
## debug level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
## the default is the same as your global application level
#level = WARNING
## path to a file to log into
#logfile = none
## maximum logfile size
#max_bytes = 30 * 1024 * 1024
## number of files to keep
#rotate = 5
## If you set backend to 'multi', add at least one section like this per
## bui-agent
#[Agent:agent1]