From b0412fec660cb704cc45aea53527763c979c63ef Mon Sep 17 00:00:00 2001 From: ziirish Date: Sat, 23 May 2020 23:07:01 +0200 Subject: [PATCH] add: last backup attempt on the clients view (fix #309) --- CHANGELOG.rst | 1 + burpui/api/clients.py | 19 ++++++----- burpui/api/misc.py | 8 ++--- burpui/api/servers.py | 4 +-- burpui/misc/backend/burp1.py | 11 ++++--- burpui/misc/backend/burp2.py | 31 ++++++++++++++---- burpui/misc/backend/interface.py | 6 +++- burpui/misc/backend/parallel.py | 56 ++++++++++++++++++++++++-------- burpui/templates/clients.html | 1 + burpui/templates/js/clients.js | 13 ++++++++ tests/unit/test_routes.py | 2 +- 11 files changed, 111 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4117d9e9..a01a20d3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,7 @@ Current - Add: new `order` keyword in ACL definitions in order to decide whether `rw` should be evaluated first or not `#305 `__ - Add: new `exclude` keyword in ACL definitions in order to exclude some clients from the rules `#305 `__ - Add: new *static templates* that allow you to create *onetime* (variables) templates `#280 `_ +- Add: return last backup attempt `#309 `_ - Add: allow to hide selected clients/servers `#282 `_ - Add: allow to delete clients data upon removal `#232 `_ - Add: allow to create clients from templates in one call `#266 `_ diff --git a/burpui/api/clients.py b/burpui/api/clients.py index e1ef6e7e..2a7196e6 100644 --- a/burpui/api/clients.py +++ b/burpui/api/clients.py @@ -94,7 +94,7 @@ class RunningClients(Resource): def __extract_running_clients(serv): try: - clients = [x['name'] for x in bui.client.get_all_clients(serv)] + clients = [x['name'] for x in bui.client.get_all_clients(serv, last_attempt=False)] except BUIserverException: clients = [] allowed = [x for x in clients if mask.is_client_allowed(current_user, x, serv)] @@ -108,7 +108,7 @@ class RunningClients(Resource): return ret else: try: - clients = [x['name'] for x in bui.client.get_all_clients(server)] + clients = [x['name'] for x in bui.client.get_all_clients(server, last_attempt=False)] except BUIserverException: clients = [] allowed = [x for x in clients if mask.is_client_allowed(current_user, x, server)] @@ -174,7 +174,7 @@ class RunningBackup(Resource): new = {} for serv in bui.client.servers: try: - clients = [x['name'] for x in bui.client.get_all_clients(serv)] + clients = [x['name'] for x in bui.client.get_all_clients(serv, last_attempt=False)] except BUIserverException: clients = [] allowed = [x for x in clients if mask.is_client_allowed(current_user, x, serv)] @@ -182,7 +182,7 @@ class RunningBackup(Resource): res = new else: try: - clients = [x['name'] for x in bui.client.get_all_clients(server)] + clients = [x['name'] for x in bui.client.get_all_clients(server, last_attempt=False)] except BUIserverException: clients = [] allowed = [x for x in clients if mask.is_client_allowed(current_user, x, server)] @@ -388,7 +388,7 @@ class ClientsReport(Resource): def _parse_clients_reports(self, res=None, server=None): if not res: try: - clients = bui.client.get_all_clients(agent=server) + clients = bui.client.get_all_clients(agent=server, last_attempt=False) except BUIserverException as e: self.abort(500, str(e)) if mask.has_filters(current_user): @@ -425,6 +425,7 @@ class ClientsStats(Resource): parser.add_argument('serverName', help='Which server to collect data from when in multi-agent mode') client_fields = ns.model('ClientsStatsSingle', { 'last': fields.DateTime(required=True, dt_format='iso8601', description='Date of last backup'), + 'last_attempt': fields.DateTime(dt_format='iso8601', description='Date of last backup attempt'), 'name': fields.String(required=True, description='Client name'), 'state': fields.LocalizedString(required=True, description='Current state of the client (idle, backup, etc.)'), 'phase': fields.String(description='Phase of the current running backup'), @@ -454,6 +455,7 @@ class ClientsStats(Resource): [ { "last": "2015-05-17 11:40:02", + "last_attempt": "2015-05-17 11:40:02", "name": "client1", "state": "idle", "phase": "phase1", @@ -464,6 +466,7 @@ class ClientsStats(Resource): }, { "last": "never", + "last_attempt": "never", "name": "client2", "state": "idle", "phase": "phase2", @@ -601,7 +604,7 @@ class AllClients(Resource): if server: try: - clients = [x['name'] for x in bui.client.get_all_clients(agent=server)] + clients = [x['name'] for x in bui.client.get_all_clients(agent=server, last_attempt=False)] except BUIserverException: clients = [] if not is_admin: @@ -613,7 +616,7 @@ class AllClients(Resource): if bui.config['STANDALONE']: try: - clients = [x['name'] for x in bui.client.get_all_clients()] + clients = [x['name'] for x in bui.client.get_all_clients(last_attempt=False)] except BUIserverException: clients = [] if not is_admin: @@ -625,7 +628,7 @@ class AllClients(Resource): clients_cache = {} for serv in bui.client.servers: try: - clients = [x['name'] for x in bui.client.get_all_clients(serv)] + clients = [x['name'] for x in bui.client.get_all_clients(serv, last_attempt=False)] clients_cache[serv] = clients except BUIserverException: clients = [] diff --git a/burpui/api/misc.py b/burpui/api/misc.py index 5905fcfc..cc2a0e31 100644 --- a/burpui/api/misc.py +++ b/burpui/api/misc.py @@ -636,7 +636,7 @@ class History(Resource): if data and server in data: clients = [{'name': x} for x in data[server].keys()] else: - clients = bui.client.get_all_clients(agent=server) + clients = bui.client.get_all_clients(agent=server, last_attempt=False) # manage ACL if has_filters: clients = [x for x in clients if mask.is_client_allowed(current_user, x['name'], server)] @@ -656,7 +656,7 @@ class History(Resource): clients_list = data.keys() else: try: - clients_list = [x['name'] for x in bui.client.get_all_clients()] + clients_list = [x['name'] for x in bui.client.get_all_clients(last_attempt=False)] except BUIserverException: clients_list = [] if has_filters: @@ -676,7 +676,7 @@ class History(Resource): for serv in bui.client.servers: if has_filters: try: - all_clients = [x['name'] for x in bui.client.get_all_clients(serv)] + all_clients = [x['name'] for x in bui.client.get_all_clients(serv, last_attempt=False)] except BUIserverException: all_clients = [] grants[serv] = [x for x in all_clients if mask.is_client_allowed(current_user, x, serv)] @@ -687,7 +687,7 @@ class History(Resource): if data and serv in data: clients = data[serv].keys() else: - clients = [x['name'] for x in bui.client.get_all_clients(agent=serv)] + clients = [x['name'] for x in bui.client.get_all_clients(agent=serv, last_attempt=False)] for cl in clients: (color, text) = self.gen_colors(cl, serv) feed = { diff --git a/burpui/api/servers.py b/burpui/api/servers.py index bee2d6d1..101fb8d3 100644 --- a/burpui/api/servers.py +++ b/burpui/api/servers.py @@ -79,7 +79,7 @@ class ServersStats(Resource): g.DONOTCACHE = True try: - clients = bui.client.servers[serv].get_all_clients(serv) + clients = bui.client.servers[serv].get_all_clients(serv, last_attempt=False) except BUIserverException: clients = [] @@ -193,7 +193,7 @@ class ServersReport(Resource): if check and not mask.is_server_allowed(current_user, serv): continue try: - clients = bui.client.get_all_clients(agent=serv) + clients = bui.client.get_all_clients(agent=serv, last_attempt=False) except BUIserverException: continue if check: diff --git a/burpui/misc/backend/burp1.py b/burpui/misc/backend/burp1.py index 5976b881..72797d6a 100644 --- a/burpui/misc/backend/burp1.py +++ b/burpui/misc/backend/burp1.py @@ -651,15 +651,15 @@ class Burp(BUIbackend): """See :func:`burpui.misc.backend.interface.BUIbackend.is_one_backup_running`""" res = [] try: - cls = self.get_all_clients() + clients = self.get_all_clients(last_attempt=False) except BUIserverException: return res - for cli in cls: - if self.is_backup_running(cli['name']): - res.append(cli['name']) + for client in clients: + if self.is_backup_running(client['name']): + res.append(client['name']) return res - def get_all_clients(self, agent=None): + def get_all_clients(self, agent=None, last_attempt=True): """See :func:`burpui.misc.backend.interface.BUIbackend.get_all_clients`""" res = [] filemap = self.status() @@ -693,6 +693,7 @@ class Burp(BUIbackend): spl = infos.split('\t') cli['last'] = int((spl[-1].split())[-1]) cli['last'] = utc_to_local(cli['last']) + cli['last_attempt'] = cli['last'] res.append(cli) return res diff --git a/burpui/misc/backend/burp2.py b/burpui/misc/backend/burp2.py index 5a4971d4..b00f0490 100644 --- a/burpui/misc/backend/burp2.py +++ b/burpui/misc/backend/burp2.py @@ -498,7 +498,7 @@ class Burp(Burp1): """ ret = [] try: - clients = self.get_all_clients() + clients = self.get_all_clients(last_attempt=False) except BUIserverException: return ret return self._do_is_one_backup_running(clients) @@ -527,19 +527,28 @@ class Burp(Burp1): return 'server crashed' return status - def _get_last_backup(self, name): + def _get_last_backup(self, name, working=True): """Return the last backup of a given client :param name: Name of the client :type name: str + :param working: Also return uncomplete backups + :type working: bool + :returns: The last backup """ try: clients = self.status('c:{}'.format(name)) client = clients['clients'][0] - return client['backups'][0] - except (KeyError, TypeError, BUIserverException): + i = 0 + while True: + ret = client['backups'][i] + if not working and "working" in ret["flags"]: + i += 1 + continue + return ret + except (KeyError, TypeError, IndexError, BUIserverException): return None def _guess_os(self, name): @@ -572,7 +581,7 @@ class Burp(Burp1): ret = OSES[-1] else: # more aggressive check - last = self._get_last_backup(name) + last = self._get_last_backup(name, False) if last: try: tree = self.get_tree(name, last['number']) @@ -587,7 +596,7 @@ class Burp(Burp1): self._os_cache[name] = ret return ret - def get_all_clients(self, agent=None): + def get_all_clients(self, agent=None, last_attempt=True): """See :func:`burpui.misc.backend.interface.BUIbackend.get_all_clients` """ @@ -603,18 +612,28 @@ class Burp(Burp1): infos = client['backups'] if cli['state'] in ['running']: cli['last'] = 'now' + cli['last_attempt'] = 'now' elif not infos: cli['last'] = 'never' + cli['last_attempt'] = 'never' else: + convert = True infos = infos[0] if self.server_version and self.server_version < BURP_STATUS_FORMAT_V2: cli['last'] = infos['timestamp'] + convert = False # only do deep inspection when server >= BURP_STATUS_FORMAT_V2 elif self.deep_inspection: logs = self.get_backup_logs(infos['number'], client['name']) cli['last'] = logs['start'] else: cli['last'] = utc_to_local(infos['timestamp']) + if last_attempt: + last_backup = self._get_last_backup(client['name']) + if convert: + cli['last_attempt'] = utc_to_local(last_backup['timestamp']) + else: + cli['last_attempt'] = last_backup['timestamp'] ret.append(cli) return ret diff --git a/burpui/misc/backend/interface.py b/burpui/misc/backend/interface.py index c513a94b..09f6fda4 100644 --- a/burpui/misc/backend/interface.py +++ b/burpui/misc/backend/interface.py @@ -501,13 +501,17 @@ class BUIbackend(object, metaclass=ABCMeta): raise NotImplementedError("Sorry, the current Backend does not implement this method!") # pragma: no cover @abstractmethod - def get_all_clients(self, agent=None): + def get_all_clients(self, agent=None, last_attempt=True): """The :func:`burpui.misc.backend.interface.BUIbackend.get_all_clients` function returns a list containing all the clients with their states. :param agent: What server to ask (only in multi-agent mode) :type agent: str + :param last_attempt: Whether to return last backup attempt or not. This requires + one more query per client hence we can disable it. + :type last_attempt: bool + :returns: A list of clients Example:: diff --git a/burpui/misc/backend/parallel.py b/burpui/misc/backend/parallel.py index 482745f1..c8396622 100644 --- a/burpui/misc/backend/parallel.py +++ b/burpui/misc/backend/parallel.py @@ -16,6 +16,7 @@ import trio import struct from asyncio import iscoroutinefunction +from functools import partial from .burp2 import Burp as Burp2 from .interface import BUIbackend, BUIBACKEND_INTERFACE_METHODS @@ -23,6 +24,7 @@ from .utils.constant import BURP_STATUS_FORMAT_V2 from ..parser.burp2 import Parser from ...exceptions import BUIserverException from ...decorators import implement, usetriorun +from ...utils import utc_to_local from ..._compat import to_unicode, to_bytes from ...tools.logging import logger @@ -493,7 +495,7 @@ class Burp(Burp2): """ ret = [] try: - clients = await self._async_get_all_clients(deep=False) + clients = await self._async_get_all_clients(deep=False, last_attempt=False) except BUIserverException: return ret return self._do_is_one_backup_running(clients) @@ -505,31 +507,43 @@ class Burp(Burp2): """ return trio.run(self._async_is_one_backup_running) - async def _async_get_last_backup(self, name): + async def _async_get_last_backup(self, name, working=True): """Return the last backup of a given client :param name: Name of the client :type name: str + :param working: Also return uncomplete backups + :type working: bool + :returns: The last backup """ try: clients = await self._async_status('c:{}'.format(name)) client = clients['clients'][0] - return client['backups'][0] - except (KeyError, BUIserverException): + i = 0 + while True: + ret = client['backups'][i] + if not working and "working" in ret["flags"]: + i += 1 + continue + return ret + except (KeyError, IndexError, BUIserverException): return None @usetriorun - def _get_last_backup(self, name): + def _get_last_backup(self, name, working=True): """Return the last backup of a given client :param name: Name of the client :type name: str + :param working: Also return uncomplete backups + :type working: bool + :returns: The last backup """ - return trio.run(self._async_get_last_backup, name) + return trio.run(self._async_get_last_backup, name, working) async def _async_guess_os(self, name): """Return the OS of the given client based on the magic *os* label @@ -561,7 +575,7 @@ class Burp(Burp2): ret = OSES[-1] else: # more aggressive check - last = await self._async_get_last_backup(name) + last = await self._async_get_last_backup(name, False) if last: try: tree = await self._async_get_tree(name, last['number']) @@ -592,7 +606,7 @@ class Burp(Burp2): """ return trio.run(self._async_guess_os, name) - async def _async_get_all_clients(self, agent=None, deep=True): + async def _async_get_all_clients(self, agent=None, deep=True, last_attempt=True): ret = [] query = await self._async_status() if not query or 'clients' not in query: @@ -606,15 +620,28 @@ class Burp(Burp2): infos = client['backups'] if cli['state'] in ['running']: cli['last'] = 'now' + cli['last_attempt'] = 'now' elif not infos: cli['last'] = 'never' + cli['last_attempt'] = 'never' else: + convert = True infos = infos[0] + if self.server_version and self.server_version < BURP_STATUS_FORMAT_V2: + cli['last'] = infos['timestamp'] + convert = False + # only do deep inspection when server >= BURP_STATUS_FORMAT_V2 if deep: logs = await self._async_get_backup_logs(infos['number'], client['name']) cli['last'] = logs['start'] else: - cli['last'] = infos['timestamp'] + cli['last'] = utc_to_local(infos['timestamp']) + if last_attempt: + last_backup = await self._async_get_last_backup(client['name']) + if convert: + cli['last_attempt'] = utc_to_local(last_backup['timestamp']) + else: + cli['last_attempt'] = last_backup['timestamp'] queue.append(cli) clients = query['clients'] @@ -626,16 +653,17 @@ class Burp(Burp2): return ret - def get_all_clients(self, agent=None): + def get_all_clients(self, agent=None, last_attempt=True): """See :func:`burpui.misc.backend.interface.BUIbackend.get_all_clients` """ # don't need async processing if burp-server < BURP_STATUS_FORMAT_V2 if not self.deep_inspection or (self.server_version and self.server_version < BURP_STATUS_FORMAT_V2): - return Burp2.get_all_clients(self) + return Burp2.get_all_clients(self, last_attempt=last_attempt) # the deep inspection can take advantage of async processing - return trio.run(self._async_get_all_clients) + callback = partial(self._async_get_all_clients, last_attempt=last_attempt) + return trio.run(callback) async def _async_get_client_status(self, name=None, agent=None): ret = {} @@ -892,11 +920,11 @@ class AsyncBurp(Burp): return await self._async_get_backup_logs(number, client, forward, deep) @implement - async def get_all_clients(self, agent=None): + async def get_all_clients(self, agent=None, last_attempt=True): """See :func:`burpui.misc.backend.interface.BUIbackend.get_all_clients` """ - return await self._async_get_all_clients() + return await self._async_get_all_clients(last_attempt=last_attempt) @implement async def get_attr(self, name, default=None, agent=None): diff --git a/burpui/templates/clients.html b/burpui/templates/clients.html index 72297fe8..45725c50 100644 --- a/burpui/templates/clients.html +++ b/burpui/templates/clients.html @@ -22,6 +22,7 @@ {{ _('Name') }} {{ _('State') }} {{ _('Last Backup') }} + {{ _('Last Attempt') }} {{ _('Labels') }} {{ _('Monitor') }} diff --git a/burpui/templates/js/clients.js b/burpui/templates/js/clients.js index 2b1907a7..ac51f4e9 100644 --- a/burpui/templates/js/clients.js +++ b/burpui/templates/js/clients.js @@ -140,6 +140,19 @@ var _clients_table = $('#table-clients').DataTable( { return data; } }, + { + data: 'last_attempt', + type: 'timestamp', + render: function (data, type, row ) { + if (type === 'filter' || type === 'sort') { + return data; + } + if (!(data in __status || data in __date)) { + return ''+moment(data, moment.ISO_8601).tz(TIMEZONE).format({{ g.date_format|tojson }})+''; + } + return data; + } + }, { data: 'labels', render: function (data, type, row) { diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py index 3f8e6c04..d8118ca7 100644 --- a/tests/unit/test_routes.py +++ b/tests/unit/test_routes.py @@ -40,7 +40,7 @@ def test_get_clients(client, mocker): mocker.patch('burpui.misc.backend.burp1.Burp.status', side_effect=mock_status) login(client, 'admin', 'admin') response = client.get(url_for('api.clients_stats')) - assert sorted(response.json, key=lambda k: k['name']) == sorted([{u'state': u'idle', u'last': u'never', u'name': u'testclient', u'phase': None, u'percent': 0, u'labels': []}], key=lambda k: k['name']) + assert sorted(response.json, key=lambda k: k['name']) == sorted([{'state': 'idle', 'last': 'never', 'last_attempt': 'never', 'name': 'testclient', 'phase': None, 'percent': 0, 'labels': []}], key=lambda k: k['name']) # def test_live_monitor(self):