rework tests/build system

2026-05-15 06:05:58 -06:00 · 2019-07-12 11:03:04 +02:00 · 2019-07-12 11:03:04 +02:00 · 43147b834c
commit 43147b834c
parent af14c99a65
7 changed files with 104 additions and 60 deletions
--- a/.gitignore
+++ b/.gitignore
@ -4,6 +4,7 @@
 burpui-dev.cfg*
 burpui/RELEASE
 devel.sh
+clean.sh
 *.egg*
 .tox
 .reports
@ -18,7 +19,5 @@ _build
 .tags
 celerybeat-schedule
 Pipfile*
-pkgs/burp-ui-sql/burpui_sql/VERSION
-pkgs/burp-ui-extra/burpui_extra/VERSION
 pkgs/burp-ui-agent/burpui_agent
 pkgs/burp-ui-monitor/burpui_monitor
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -3,6 +3,10 @@ variables:
  GIT_SUBMODULE_STRATEGY: recursive
  DOCKER_DRIVER: overlay2
  DOCKER_HOST: tcp://localhost:2375
+  BURP_VERSION: 2.2.18
+  PG_VERSION: 10
+  FLAKE8_IGNORE: E501,E722,W605,W504
+  SRC_DIR: burpui

 image: docker:git

@ -14,20 +18,43 @@ stages:
  - build
  - deploy

-test:lint:
+test:lint:3.6:
  stage: test
-  image: python:3.6
+  image: python:3.6-alpine
  script:
  - pip install flake8 pylint
-  - make flake8
+  - flake8 --ignore=$FLAKE8_IGNORE $SRC_DIR
  tags:
  - lint
  except:
  - tags

-test:py3.6:
+test:lint:3.7:
  stage: test
-  image: python:3.6
+  image: python:3.7-alpine
+  script:
+  - pip install flake8 pylint
+  - flake8 --ignore=$FLAKE8_IGNORE $SRC_DIR
+  tags:
+  - lint
+  except:
+  - tags
+
+test:lint:3.8:
+  stage: test
+  image: python:3.8-rc-alpine
+  script:
+  - pip install flake8 pylint
+  - flake8 --ignore=$FLAKE8_IGNORE $SRC_DIR
+  tags:
+  - lint
+  except:
+  - tags
+  allow_failure: true
+
+test:py:3.6:
+  stage: test
+  image: python:3.6-alpine
  script:
  - pip install tox
  - mkdir .reports
@ -40,11 +67,42 @@ test:py3.6:
    reports:
      junit: .reports/burpui.junit.xml

+test:py:3.7:
+  stage: test
+  image: python:3.7-alpine
+  script:
+  - pip install tox
+  - mkdir .reports
+  - tox -e py37
+  tags:
+  - docker
+  except:
+  - tags
+  artifacts:
+    reports:
+      junit: .reports/burpui.junit.xml
+
+test:py:3.8:
+  stage: test
+  image: python:3.8-rc-alpine
+  script:
+  - pip install tox
+  - mkdir .reports
+  - tox -e py38
+  tags:
+  - docker
+  except:
+  - tags
+  artifacts:
+    reports:
+      junit: .reports/burpui.junit.xml
+  allow_failure: true
+
 build:py3:
  stage: build
-  image: python:3.6
+  image: python:3.6-alpine
  script:
-  - /bin/bash tests/run_build.sh
+  - tests/build.sh
  tags:
  - build
  only:
@ -59,10 +117,10 @@ build:py3:

 build:doc:
  stage: build
-  image: python:3.6
+  image: python:3.6-alpine
  script:
  - pip install -U .[rtd]
-  - make doc
+  - cd doc && make html
  tags:
  - build
  only:
@ -78,11 +136,11 @@ build:docker:latest:
  script:
  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
  - docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest -f docker/Dockerfile .
-  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 .)
-  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18 .)
+  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION .)
+  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION .)
  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:latest
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION
  tags:
  - registry
  only:
@ -93,11 +151,11 @@ build:docker:release:
  script:
  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
  - docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG -f docker/Dockerfile .
-  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 .)
-  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18 .)
+  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION .)
+  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION .)
  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:$CI_COMMIT_TAG
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION
  only:
  - tags
  tags:
@ -108,11 +166,11 @@ build:docker:stable:
  script:
  - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
  - docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:stable -f docker/Dockerfile .
-  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10 .)
-  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18 .)
+  - (cd docker/demo/docker-pg && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION .)
+  - (cd docker/components/docker-burp && docker build --pull -t $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION .)
  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME:stable
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:10
-  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:2.2.18
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/pgsql:$PG_VERSION
+  - docker push $CI_REGISTRY/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME/burp:$BURP_VERSION
  only:
  - stable@ziirish/burp-ui
  tags:
--- a/requirements.txt
+++ b/requirements.txt
@ -1,17 +1,16 @@
 trio==0.11.0
-Flask==1.0.2
+Flask==1.1.1
 Flask-Login==0.4.1
 Flask-Bower==1.3.0
 Flask-Babel==0.12.2
 Flask-WTF==0.14.2
 flask-restplus==0.12.1
-Flask-Caching==1.6.0
+Flask-Caching==1.7.2
 WTForms==2.1
-arrow==0.13.1
+arrow==0.14.2
 pluginbase==1.0.0
 tzlocal==1.5.1
 pyOpenSSL==19.0.0
 configobj==5.0.6
 async_generator
 Click==7.0
-werkzeug>=0.15.0
--- a/setup.py
+++ b/setup.py
@ -301,6 +301,7 @@ setup(
        (os.path.join(contrib, 'gunicorn'), ['contrib/gunicorn/burpui_gunicorn.py']),
        (os.path.join(contrib, 'systemd'), ['contrib/systemd/bui-agent.service', 'contrib/systemd/bui-celery.service', 'contrib/systemd/bui-gunicorn.service']),
    ] + migrations,
+    python_requires=">=3.6",
    install_requires=requires,
    extras_require={
        'ldap_authentication': ['ldap3'],
--- a/tests/build.sh
+++ b/tests/build.sh
@ -0,0 +1,19 @@
+#!/bin/sh
+
+pip install wheel
+
+echo "building dist"
+python setup.py sdist bdist_wheel bdist_egg
+mkdir meta
+cd pkgs
+for pkg in *
+do
+    [ "$pkg" = "burp-ui-tpl" -o -f "$pkg" ] && continue
+    cd $pkg
+    python setup.py sdist bdist_wheel bdist_egg
+    find dist -type f -exec cp "{}" ../../meta/ \;
+    cd ..
+done
+cd ..
+
+exit 0
--- a/tests/run_build.sh
+++ b/tests/run_build.sh
@ -1,32 +0,0 @@
-#!/bin/bash
-
-PYTHON=$(which python)
-
-echo "test requirements"
-
-[ -x "$PYTHON" ] && {
-    echo "python seems to be installed"
-} || {
-    echo "python is missing..."
-    exit 1
-}
-
-pip install -r requirements.txt
-pip install wheel
-
-
-echo "building dist"
-$PYTHON setup.py sdist bdist_wheel bdist_egg
-mkdir meta
-cd pkgs
-for pkg in *
-do
-    [ "$pkg" = "burp-ui-tpl" -o -f "$pkg" ] && continue
-    cd $pkg
-    $PYTHON setup.py sdist bdist_wheel bdist_egg
-    find dist -exec cp "{}" ../../meta/ \;
-    cd ..
-done
-cd ..
-
-exit 0
--- a/tox.ini
+++ b/tox.ini
@ -1,5 +1,5 @@
 [tox]
-envlist = py{36}
+envlist = py{36,37,38}

 [testenv]
 description = Run burp-ui unit tests