diff --git a/burpui/agent.py b/burpui/agent.py index 5b027b50..efa4b80f 100644 --- a/burpui/agent.py +++ b/burpui/agent.py @@ -8,6 +8,8 @@ import json import logging import traceback +from gevent.coros import RLock +from gevent.pool import Pool from gevent.server import StreamServer from logging.handlers import RotatingFileHandler from .exceptions import BUIserverException @@ -25,6 +27,8 @@ G_PASSWORD = u'password' DISCLOSURE = 5 +lock = RLock() + class BurpHandler(BUIbackend): # These functions MUST be implemented because we inherit an abstract class. @@ -128,10 +132,11 @@ class BUIAgent(BUIbackend, BUIlogging): self.password = self.conf.safe_get('password') self.cli = BurpHandler(self.vers, self.logger, self.conf) + pool = Pool(10000) if not self.ssl: - self.server = StreamServer((self.bind, self.port), self.handle) + self.server = StreamServer((self.bind, self.port), self.handle, spawn=pool) else: - self.server = StreamServer((self.bind, self.port), self.handle, keyfile=self.sslkey, certfile=self.sslcert) + self.server = StreamServer((self.bind, self.port), self.handle, keyfile=self.sslkey, certfile=self.sslcert, spawn=pool) def run(self): try: @@ -141,129 +146,130 @@ class BUIAgent(BUIbackend, BUIlogging): def handle(self, request, address): """self.request is the client connection""" - try: - self.request = request - err = None - res = '' - lengthbuf = self.request.recv(8) - length, = struct.unpack('!Q', lengthbuf) - data = self.recvall(length) - self._logger('info', 'recv: {}'.format(data)) - txt = data.decode('UTF-8') - if txt == 'RE': - return - j = json.loads(txt) - if j['password'] != self.password: - self._logger('warning', '-----> Wrong Password <-----') - self.request.sendall(b'KO') - return + with lock: try: - if j['func'] == 'proxy_parser': - parser = self.cli.get_parser() - if j['args']: - res = json.dumps(getattr(parser, j['method'])(**j['args'])) - else: - res = json.dumps(getattr(parser, j['method'])()) - elif j['func'] == 'restore_files': - res, err = getattr(self.cli, j['func'])(**j['args']) - if err: - self.request.sendall(b'ER') - self.request.sendall(struct.pack('!Q', len(err))) - self.request.sendall(err.encode('UTF-8')) - self._logger('error', 'Restoration failed') - return - elif j['func'] == 'get_file': - path = j['path'] - path = os.path.normpath(path) - err = None - if not path.startswith('/'): - err = 'The path must be absolute! ({})'.format(path) - if not path.startswith(self.cli.tmpdir): - err = 'You are not allowed to access this path: ' \ - '({})'.format(path) - if err: - self.request.sendall(b'ER') - self.request.sendall(struct.pack('!Q', len(err))) - self.request.sendall(err.encode('UTF-8')) - self._logger('error', err) - return - size = os.path.getsize(path) - self.request.sendall(b'OK') - self.request.sendall(struct.pack('!Q', size)) - with open(path, 'rb') as f: - buf = f.read(1024) - while buf: - self._logger('info', 'sending {} Bytes'.format(len(buf))) - self.request.sendall(buf) + self.request = request + err = None + res = '' + lengthbuf = self.request.recv(8) + length, = struct.unpack('!Q', lengthbuf) + data = self.recvall(length) + self._logger('info', 'recv: {}'.format(data)) + txt = data.decode('UTF-8') + if txt == 'RE': + return + j = json.loads(txt) + if j['password'] != self.password: + self._logger('warning', '-----> Wrong Password <-----') + self.request.sendall(b'KO') + return + try: + if j['func'] == 'proxy_parser': + parser = self.cli.get_parser() + if j['args']: + res = json.dumps(getattr(parser, j['method'])(**j['args'])) + else: + res = json.dumps(getattr(parser, j['method'])()) + elif j['func'] == 'restore_files': + res, err = getattr(self.cli, j['func'])(**j['args']) + if err: + self.request.sendall(b'ER') + self.request.sendall(struct.pack('!Q', len(err))) + self.request.sendall(err.encode('UTF-8')) + self._logger('error', 'Restoration failed') + return + elif j['func'] == 'get_file': + path = j['path'] + path = os.path.normpath(path) + err = None + if not path.startswith('/'): + err = 'The path must be absolute! ({})'.format(path) + if not path.startswith(self.cli.tmpdir): + err = 'You are not allowed to access this path: ' \ + '({})'.format(path) + if err: + self.request.sendall(b'ER') + self.request.sendall(struct.pack('!Q', len(err))) + self.request.sendall(err.encode('UTF-8')) + self._logger('error', err) + return + size = os.path.getsize(path) + self.request.sendall(b'OK') + self.request.sendall(struct.pack('!Q', size)) + with open(path, 'rb') as f: buf = f.read(1024) - os.unlink(path) - lengthbuf = self.request.recv(8) - length, = struct.unpack('!Q', lengthbuf) - data = self.recvall(length) - txt = data.decode('UTF-8') - if txt == 'RE': - return - elif j['func'] == 'del_file': - path = j['path'] - path = os.path.normpath(path) - err = None - if not path.startswith('/'): - err = 'The path must be absolute! ({})'.format(path) - if not path.startswith(self.cli.tmpdir): - err = 'You are not allowed to access this path: ' \ - '({})'.format(path) - if err: - self.request.sendall(b'ER') - self.request.sendall(struct.pack('!Q', len(err))) - self.request.sendall(err.encode('UTF-8')) - self._logger('error', err) - return - res = json.dumps(False) - if os.path.isfile(path): + while buf: + self._logger('info', 'sending {} Bytes'.format(len(buf))) + self.request.sendall(buf) + buf = f.read(1024) os.unlink(path) - res = json.dumps(True) - else: - if j['args']: - if 'pickled' in j and j['pickled']: - # de-serialize arguments if needed - import hmac - import hashlib - from base64 import b64decode - pickles = j['args'] - key = u'{}{}'.format(self.password, j['func']) - key = key.encode(encoding='utf-8') - bytes_pickles = pickles.encode(encoding='utf-8') - digest = hmac.new(key, bytes_pickles, hashlib.sha1).hexdigest() - if digest != j['digest']: - raise BUIserverException('Integrity check failed') - j['args'] = pickle.loads(b64decode(pickles)) - res = json.dumps(getattr(self.cli, j['func'])(**j['args'])) + lengthbuf = self.request.recv(8) + length, = struct.unpack('!Q', lengthbuf) + data = self.recvall(length) + txt = data.decode('UTF-8') + if txt == 'RE': + return + elif j['func'] == 'del_file': + path = j['path'] + path = os.path.normpath(path) + err = None + if not path.startswith('/'): + err = 'The path must be absolute! ({})'.format(path) + if not path.startswith(self.cli.tmpdir): + err = 'You are not allowed to access this path: ' \ + '({})'.format(path) + if err: + self.request.sendall(b'ER') + self.request.sendall(struct.pack('!Q', len(err))) + self.request.sendall(err.encode('UTF-8')) + self._logger('error', err) + return + res = json.dumps(False) + if os.path.isfile(path): + os.unlink(path) + res = json.dumps(True) else: - res = json.dumps(getattr(self.cli, j['func'])()) - self._logger('info', 'result: {}'.format(res)) - self.request.sendall(b'OK') - # should not happen - except Exception as e: - raise BUIserverException(str(e)) - except BUIserverException as e: - self.request.sendall(b'ER') - res = str(e) - self._logger('error', 'Forwarding Exception: {}'.format(res)) + if j['args']: + if 'pickled' in j and j['pickled']: + # de-serialize arguments if needed + import hmac + import hashlib + from base64 import b64decode + pickles = j['args'] + key = u'{}{}'.format(self.password, j['func']) + key = key.encode(encoding='utf-8') + bytes_pickles = pickles.encode(encoding='utf-8') + digest = hmac.new(key, bytes_pickles, hashlib.sha1).hexdigest() + if digest != j['digest']: + raise BUIserverException('Integrity check failed') + j['args'] = pickle.loads(b64decode(pickles)) + res = json.dumps(getattr(self.cli, j['func'])(**j['args'])) + else: + res = json.dumps(getattr(self.cli, j['func'])()) + self._logger('info', 'result: {}'.format(res)) + self.request.sendall(b'OK') + # should not happen + except Exception as e: + raise BUIserverException(str(e)) + except BUIserverException as e: + self.request.sendall(b'ER') + res = str(e) + self._logger('error', 'Forwarding Exception: {}'.format(res)) + self.request.sendall(struct.pack('!Q', len(res))) + self.request.sendall(res.encode('UTF-8')) + return self.request.sendall(struct.pack('!Q', len(res))) self.request.sendall(res.encode('UTF-8')) - return - self.request.sendall(struct.pack('!Q', len(res))) - self.request.sendall(res.encode('UTF-8')) - except AttributeError as e: - self._logger('warning', '{}\nWrong method => {}'.format(traceback.format_exc(), str(e))) - self.request.sendall(b'KO') - except Exception as e: - self._logger('error', '!!! {} !!!\n{}'.format(str(e), traceback.format_exc())) - finally: - try: - self.request.close() + except AttributeError as e: + self._logger('warning', '{}\nWrong method => {}'.format(traceback.format_exc(), str(e))) + self.request.sendall(b'KO') except Exception as e: self._logger('error', '!!! {} !!!\n{}'.format(str(e), traceback.format_exc())) + finally: + try: + self.request.close() + except Exception as e: + self._logger('error', '!!! {} !!!\n{}'.format(str(e), traceback.format_exc())) def recvall(self, length=1024): buf = b'' diff --git a/burpui/misc/backend/multi.py b/burpui/misc/backend/multi.py index 22c9e287..fd802b57 100644 --- a/burpui/misc/backend/multi.py +++ b/burpui/misc/backend/multi.py @@ -15,6 +15,18 @@ from ...exceptions import BUIserverException from ..._compat import pickle from ...utils import implement +try: + from gevent.coros import RLock + lock = RLock() +except ImportError: + class DummyLock(): + def __enter__(self): + pass + + def __exit__(self, type, value, traceback): + pass + + lock = DummyLock() INTERFACE_METHODS = BUIbackend.__abstractmethods__ PARSER_INTERFACE_METHODS = BUIparser.__abstractmethods__ @@ -377,80 +389,81 @@ class NClient(BUIbackend): def do_command(self, data=None, restarted=False): """Send a command to the remote agent""" - res = '[]' - err = None - if not data: - raise BUIserverException('Missing data') - try: - data['password'] = self.password - # manage long running operations - if data['func'] in ['restore_files', 'get_file', 'del_file']: - self.connected = False - self.conn(notimeout=True) - else: - self.conn() - if not self.connected: - raise BUIserverException('Failed to connect to agent') - raw = json.dumps(data) - length = len(raw) - self.sock.sendall(struct.pack('!Q', length)) - self.sock.sendall(raw.encode('UTF-8')) - self.logger.debug("Sending: {}".format(raw)) - tmp = self.sock.recv(2).decode('UTF-8') - self.logger.debug("recv: '{}'".format(tmp)) - if 'ER' == tmp: + with lock: + res = '[]' + err = None + if not data: + raise BUIserverException('Missing data') + try: + data['password'] = self.password + # manage long running operations + if data['func'] in ['restore_files', 'get_file', 'del_file']: + self.connected = False + self.conn(notimeout=True) + else: + self.conn() + if not self.connected: + raise BUIserverException('Failed to connect to agent') + raw = json.dumps(data) + length = len(raw) + self.sock.sendall(struct.pack('!Q', length)) + self.sock.sendall(raw.encode('UTF-8')) + self.logger.debug("Sending: {}".format(raw)) + tmp = self.sock.recv(2).decode('UTF-8') + self.logger.debug("recv: '{}'".format(tmp)) + if 'ER' == tmp: + lengthbuf = self.sock.recv(8) + length, = struct.unpack('!Q', lengthbuf) + err = self.recvall(length).decode('UTF-8') + raise BUIserverException(err) + if 'OK' != tmp: + self.logger.debug('Ooops, unsuccessful!') + return res + self.logger.debug("Data sent successfully") + if data['func'] == 'get_file': + self.connected = False + return self.sock lengthbuf = self.sock.recv(8) length, = struct.unpack('!Q', lengthbuf) - err = self.recvall(length).decode('UTF-8') - raise BUIserverException(err) - if 'OK' != tmp: - self.logger.debug('Ooops, unsuccessful!') - return res - self.logger.debug("Data sent successfully") - if data['func'] == 'get_file': - self.connected = False - return self.sock - lengthbuf = self.sock.recv(8) - length, = struct.unpack('!Q', lengthbuf) - res = self.recvall(length).decode('UTF-8') - except IOError as e: - if not restarted and e.errno == errno.EPIPE: - self.connected = False - self.logger.warning('Broken pipe, restarting the request') - return self.do_command(data, True) - elif e.errno == errno.ECONNRESET: - self.connected = False - self.logger.error('!!! {} !!!\nPlease check your SSL configuration on both sides!'.format(str(e))) - else: - self.logger.error('!!! {} !!!\n{}'.format(str(e), traceback.format_exc())) - raise e - except socket.timeout as e: - if self.app.gunicorn and not restarted: - self.connected = False - self.logger.warning('Socket timed-out, restarting the request') - return self.do_command(data, True) - self.logger.error('!!! {} !!!\n{}'.format(str(e), traceback.format_exc())) - raise e - # catch all - except Exception as e: - self.logger.error('!!! {} !!!\n{}'.format(str(e), traceback.format_exc())) - if data['func'] == 'restore_files': - err = str(e) - elif isinstance(e, BUIserverException): + res = self.recvall(length).decode('UTF-8') + except IOError as e: + if not restarted and e.errno == errno.EPIPE: + self.connected = False + self.logger.warning('Broken pipe, restarting the request') + return self.do_command(data, True) + elif e.errno == errno.ECONNRESET: + self.connected = False + self.logger.error('!!! {} !!!\nPlease check your SSL configuration on both sides!'.format(str(e))) + else: + self.logger.error('!!! {} !!!\n{}'.format(str(e), traceback.format_exc())) raise e - else: - raise BUIserverException(str(e)) - finally: - if self.connected: - self.sock.close() - self.connected = False + except socket.timeout as e: + if self.app.gunicorn and not restarted: + self.connected = False + self.logger.warning('Socket timed-out, restarting the request') + return self.do_command(data, True) + self.logger.error('!!! {} !!!\n{}'.format(str(e), traceback.format_exc())) + raise e + # catch all + except Exception as e: + self.logger.error('!!! {} !!!\n{}'.format(str(e), traceback.format_exc())) + if data['func'] == 'restore_files': + err = str(e) + elif isinstance(e, BUIserverException): + raise e + else: + raise BUIserverException(str(e)) + finally: + if self.connected: + self.sock.close() + self.connected = False - if data['func'] == 'restore_files': - if err: - res = None - return res, err + if data['func'] == 'restore_files': + if err: + res = None + return res, err - return res + return res def recvall(self, length=1024): """Read the answer of the agent""" diff --git a/docker/docker-burp1/assets/setup/install b/docker/docker-burp1/assets/setup/install index c26007da..2ac9c524 100755 --- a/docker/docker-burp1/assets/setup/install +++ b/docker/docker-burp1/assets/setup/install @@ -100,6 +100,7 @@ EOF cat ${CONFIG_DIR}/burp/burp.conf >/tmp/burp.conf chown burpui: /tmp/burp.conf +sed -i "s/^max_status_children.*$/max_status_children = 10000/" /etc/burp/burp-server.conf echo "restore_client = agent" >>/etc/burp/burp-server.conf cp ${CONFIG_DIR}/burp-ui/buiagent.cfg /etc/burp/buiagent.cfg diff --git a/docker/docker-burp2/assets/setup/install b/docker/docker-burp2/assets/setup/install index 76e187a3..07fbc8c2 100755 --- a/docker/docker-burp2/assets/setup/install +++ b/docker/docker-burp2/assets/setup/install @@ -98,6 +98,7 @@ chown burpui: /tmp/burp.conf cat ${CONFIG_DIR}/burp/CA.cnf >/etc/burp/CA.cnf +sed -i "s/^max_status_children.*$/max_status_children = 10000/" /etc/burp/burp-server.conf echo "restore_client = agent" >>/etc/burp/burp-server.conf echo "monitor_browse_cache = 1" >>/etc/burp/burp-server.conf