From 326bb97a5d58d7e78713b475a71746e6a3896744 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 11:22:02 +0200 Subject: [PATCH 01/18] added CI metrics stage & updated Python image --- .gitlab-ci.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5ff4c2..6d1c19c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ # GitLab CI in conjunction with GitLab Runner can use Docker Engine to test and build any application. # Docker, when used with GitLab CI, runs each job in a separate and isolated container using the predefined image that is set up in .gitlab-ci.yml. # In this case we use the latest python docker image to build and test this project. -image: python:3.5-slim +image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest # cache is used to specify a list of files and directories which should be cached between jobs. You can only use paths that are within the project workspace. # If cache is defined outside the scope of jobs, it means it is set globally and all jobs will use that definition @@ -16,6 +16,7 @@ before_script: stages: - test + - linting - deploy # The YAML file defines a set of jobs with constraints stating when they should be run. @@ -90,4 +91,20 @@ pages: artifacts: paths: - public - expire_in: 30 days \ No newline at end of file + expire_in: 30 days + +create ci metrics: + stage: .post + image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest + when: always + tags: + - docker-executor + script: + # Gitlab CI badges creation: START + - apt-get -y update + - apt-get install -y curl --no-install-recommends + - curl -s https://gitlab.com/ska-telescope/ci-metrics-utilities/raw/master/scripts/ci-badges-func.sh | sh + # Gitlab CI badges creation: END + artifacts: + paths: + - ./build \ No newline at end of file -- GitLab From 165867b5e9908c6bd96380a6e7da2a6afb678ede Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 12:10:59 +0200 Subject: [PATCH 02/18] added unit test output file & changed number of processors for linting to auto --- .pylintrc | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pylintrc b/.pylintrc index 56f86d6..eae6a06 100644 --- a/.pylintrc +++ b/.pylintrc @@ -19,7 +19,7 @@ ignore-patterns= # Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the # number of processors available to use. -jobs=1 +jobs=0 # Control the amount of potential inferred values when inferring a single # object. This can help the performance when dealing with large functions or diff --git a/setup.cfg b/setup.cfg index 140dc01..95ac321 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,7 @@ source = ska_python_skeleton [tool:pytest] testpaths = tests -addopts = --cov --json-report --json-report-file=htmlcov/report.json --cov-report term --cov-report html --cov-report xml --pylint --pylint-error-types=EF +addopts = --cov --json-report --json-report-file=htmlcov/report.json --cov-report term --cov-report html --cov-report xml --pylint --pylint-error-types=EF --junitxml=/build/reports/unit-tests.xml # Define `python setup.py build_sphinx` [build_sphinx] -- GitLab From d2f88c0d3873babcf4602fc18deb4eafa6115c21 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 12:27:56 +0200 Subject: [PATCH 03/18] Added a few things to linting step --- .gitlab-ci.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d1c19c..5331310 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,11 +72,19 @@ code_quality: paths: [gl-code-quality-report.json] -code_analysis: +linting: + stage: linting + image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest + tags: + - docker-executor script: - pipenv run pylint ska_python_skeleton - pipenv run pylint tests - + artifacts: + paths: + - ./build + + pages: stage: deploy # tags: -- GitLab From b5d39ceea6651c6a73aa0d54a2ebbb6927a2f264 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 12:49:39 +0200 Subject: [PATCH 04/18] Attempted linting fix --- .gitlab-ci.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5331310..8835ada 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,10 +29,10 @@ test: # - docker-executor script: - pipenv run python setup.py test - - cp coverage.xml htmlcov/ + - cp coverage.xml ./build/reports/code-coverage.xml artifacts: paths: - - htmlcov + - build list_dependencies: stage: test @@ -72,19 +72,18 @@ code_quality: paths: [gl-code-quality-report.json] -linting: - stage: linting +code_analysis: image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest tags: - docker-executor + stage: linting script: - pipenv run pylint ska_python_skeleton - pipenv run pylint tests - artifacts: + artifacts: paths: - ./build - pages: stage: deploy # tags: -- GitLab From cfe9ff7e37bb2b3ffed5e26a134fd4e3c723ee56 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 15:00:00 +0200 Subject: [PATCH 05/18] Removed the . --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8835ada..7046b0d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,7 @@ test: # - docker-executor script: - pipenv run python setup.py test - - cp coverage.xml ./build/reports/code-coverage.xml + - cp coverage.xml /build/reports/code-coverage.xml artifacts: paths: - build -- GitLab From c3022c402c62d5ad883b5e15704fd4e2ea68f308 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 15:02:57 +0200 Subject: [PATCH 06/18] temporarily removed code quality step --- .gitlab-ci.yml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7046b0d..331d668 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,27 +49,27 @@ list_dependencies: paths: - public -code_quality: - tags: - - docker-executor - image: docker:stable - variables: - DOCKER_DRIVER: overlay2 - allow_failure: true - services: - - docker:stable-dind - before_script: - - ls -la - script: - - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') - - docker run - --env SOURCE_CODE="$PWD" - --volume "$PWD":/code - --volume /var/run/docker.sock:/var/run/docker.sock - "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code - - more gl-code-quality-report.json - artifacts: - paths: [gl-code-quality-report.json] +#code_quality: +# tags: +# - docker-executor +# image: docker:stable +# variables: +# DOCKER_DRIVER: overlay2 +# allow_failure: true +# services: +# - docker:stable-dind +# before_script: +# - ls -la +# script: +# - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') +# - docker run +# --env SOURCE_CODE="$PWD" +# --volume "$PWD":/code +# --volume /var/run/docker.sock:/var/run/docker.sock +# "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code +# - more gl-code-quality-report.json +# artifacts: +# paths: [gl-code-quality-report.json] code_analysis: -- GitLab From ce27e1420daba4f8147cc933ae14b1ac0c4c7d1d Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 15:04:59 +0200 Subject: [PATCH 07/18] Always do linting --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 331d668..5cf1214 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,6 +80,7 @@ code_analysis: script: - pipenv run pylint ska_python_skeleton - pipenv run pylint tests + when: always artifacts: paths: - ./build -- GitLab From 85b4c64763b4a814fd385df3c3dd49bf95f337ff Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 15:39:05 +0200 Subject: [PATCH 08/18] Create ./build directory --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5cf1214..eeee861 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,10 +29,12 @@ test: # - docker-executor script: - pipenv run python setup.py test - - cp coverage.xml /build/reports/code-coverage.xml + - mkdir ./build + - mkdir ./build/reports + - cp coverage.xml ./build/reports/code-coverage.xml artifacts: paths: - - build + - ./build list_dependencies: stage: test @@ -80,6 +82,7 @@ code_analysis: script: - pipenv run pylint ska_python_skeleton - pipenv run pylint tests + - cp when: always artifacts: paths: -- GitLab From 97cf329a4f57b7297db911108049b3f49e467130 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 15:39:58 +0200 Subject: [PATCH 09/18] small fix --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eeee861..d4b8720 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -82,7 +82,7 @@ code_analysis: script: - pipenv run pylint ska_python_skeleton - pipenv run pylint tests - - cp + - cp /build/reports/unit-tests.xml ./build/reports/unit-tests.xml when: always artifacts: paths: -- GitLab From 8957562c27aa40e9efdcd4fb52a44e6e1760e2c1 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 15:51:18 +0200 Subject: [PATCH 10/18] another attempt at fixing the target for unit test result XML file --- .gitlab-ci.yml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d4b8720..a938e22 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,6 +32,7 @@ test: - mkdir ./build - mkdir ./build/reports - cp coverage.xml ./build/reports/code-coverage.xml + - cp unit-tests.xml ./build/reports/unit-tests.xml artifacts: paths: - ./build @@ -82,7 +83,6 @@ code_analysis: script: - pipenv run pylint ska_python_skeleton - pipenv run pylint tests - - cp /build/reports/unit-tests.xml ./build/reports/unit-tests.xml when: always artifacts: paths: diff --git a/setup.cfg b/setup.cfg index 95ac321..4d411a1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,7 @@ source = ska_python_skeleton [tool:pytest] testpaths = tests -addopts = --cov --json-report --json-report-file=htmlcov/report.json --cov-report term --cov-report html --cov-report xml --pylint --pylint-error-types=EF --junitxml=/build/reports/unit-tests.xml +addopts = --cov --json-report --json-report-file=htmlcov/report.json --cov-report term --cov-report html --cov-report xml --pylint --pylint-error-types=EF --junitxml=unit-tests.xml # Define `python setup.py build_sphinx` [build_sphinx] -- GitLab From 43e973965f76af9f510b69456433d3467354edae Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 16:16:39 +0200 Subject: [PATCH 11/18] mv instead of cp --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a938e22..d5929e0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,8 +31,8 @@ test: - pipenv run python setup.py test - mkdir ./build - mkdir ./build/reports - - cp coverage.xml ./build/reports/code-coverage.xml - - cp unit-tests.xml ./build/reports/unit-tests.xml + - mv coverage.xml ./build/reports/code-coverage.xml + - mv unit-tests.xml ./build/reports/unit-tests.xml artifacts: paths: - ./build -- GitLab From e1aebdeee0e78a69ed55a7124efaf6f108da4f30 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Wed, 11 Dec 2019 16:24:00 +0200 Subject: [PATCH 12/18] uncommented docker-executor --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5929e0..bf6a1b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,8 +90,8 @@ code_analysis: pages: stage: deploy -# tags: -# - docker-executor + tags: + - docker-executor dependencies: - test script: -- GitLab From cce54ecdedce6cb7238966ee1e1432141491181d Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Thu, 12 Dec 2019 10:21:16 +0200 Subject: [PATCH 13/18] directly specified unittest to be stored in correct folder --- .gitlab-ci.yml | 4 +--- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf6a1b1..c4f4ab3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,7 +22,6 @@ stages: # The YAML file defines a set of jobs with constraints stating when they should be run. # You can specify an unlimited number of jobs which are defined as top-level elements with an arbitrary name and always have to contain at least the script clause. # In this case we have only the test job which produce an artifacts (it must be placed into a directory called "public") -# It is also specified that only the master branch will be subject of this job. test: stage: test # tags: @@ -32,7 +31,6 @@ test: - mkdir ./build - mkdir ./build/reports - mv coverage.xml ./build/reports/code-coverage.xml - - mv unit-tests.xml ./build/reports/unit-tests.xml artifacts: paths: - ./build @@ -75,7 +73,7 @@ list_dependencies: # paths: [gl-code-quality-report.json] -code_analysis: +linting: image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest tags: - docker-executor diff --git a/setup.cfg b/setup.cfg index 4d411a1..3f0083c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,7 @@ source = ska_python_skeleton [tool:pytest] testpaths = tests -addopts = --cov --json-report --json-report-file=htmlcov/report.json --cov-report term --cov-report html --cov-report xml --pylint --pylint-error-types=EF --junitxml=unit-tests.xml +addopts = --cov --json-report --json-report-file=htmlcov/report.json --cov-report term --cov-report html --cov-report xml --pylint --pylint-error-types=EF --junitxml=./build/reports/unit-tests.xml # Define `python setup.py build_sphinx` [build_sphinx] -- GitLab From c4f18e1b55d45d41728a595ab70b4a76c1863917 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Thu, 12 Dec 2019 10:27:02 +0200 Subject: [PATCH 14/18] removed mkdir ./build --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c4f4ab3..d5822a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,8 +28,6 @@ test: # - docker-executor script: - pipenv run python setup.py test - - mkdir ./build - - mkdir ./build/reports - mv coverage.xml ./build/reports/code-coverage.xml artifacts: paths: -- GitLab From 141f2a41ef1786db450fc6bd1f7b6c521b5f16dd Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Thu, 12 Dec 2019 10:42:55 +0200 Subject: [PATCH 15/18] test persistence of htmlcov directory --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5822a7..da0e9e7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,6 +32,7 @@ test: artifacts: paths: - ./build + - htmlcov list_dependencies: stage: test -- GitLab From 574442607f3bbcfe056a1de259103e63b7831a31 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Thu, 12 Dec 2019 11:26:20 +0200 Subject: [PATCH 16/18] pipe output to linting.xml. Uncommented second linting step. --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da0e9e7..28e95a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -78,8 +78,8 @@ linting: - docker-executor stage: linting script: - - pipenv run pylint ska_python_skeleton - - pipenv run pylint tests + - pipenv run pylint ska_python_skeleton > ./build/reports/linting.xml +# - pipenv run pylint tests when: always artifacts: paths: @@ -95,6 +95,7 @@ pages: - ls -la - mkdir .public - cp -r htmlcov/* .public + - rm -rf htmlcov - mv .public public artifacts: paths: -- GitLab From 0bcda25317f5c18bcc1260df11fb9867cf4a6632 Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Thu, 12 Dec 2019 12:40:22 +0200 Subject: [PATCH 17/18] Minimal Makefile implimentation --- .gitlab-ci.yml | 14 ++++++++------ Makefile | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) create mode 100644 Makefile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28e95a9..c24ed0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,8 +7,7 @@ image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest # If cache is defined outside the scope of jobs, it means it is set globally and all jobs will use that definition cache: paths: - -# before_script is used to define the command that should be run before all jobs, including deploy jobs, but after the restoration of artifacts. +# before_script is used to define the command that should be run before all jobs, including deploy jobs, but after the restoration of artifacts. # This can be an array or a multi-line string. before_script: - pip install pipenv @@ -20,8 +19,11 @@ stages: - deploy # The YAML file defines a set of jobs with constraints stating when they should be run. -# You can specify an unlimited number of jobs which are defined as top-level elements with an arbitrary name and always have to contain at least the script clause. -# In this case we have only the test job which produce an artifacts (it must be placed into a directory called "public") +# You can specify an unlimited number of jobs which are defined as top-level elements with an arbitrary name and always +# have to contain at least the script clause. +# In this case we have only the test job which produces a coverage report and the unittest output (see setup.cfg), and +# the coverage xml report is moved to the reports directory while the html output is persisted for use by the pages +# job. TODO: possibly a candidate for refactor / renaming later on. test: stage: test # tags: @@ -49,6 +51,7 @@ list_dependencies: paths: - public +# TODO: Not working properly at the moment - commented out temporarily #code_quality: # tags: # - docker-executor @@ -78,8 +81,7 @@ linting: - docker-executor stage: linting script: - - pipenv run pylint ska_python_skeleton > ./build/reports/linting.xml -# - pipenv run pylint tests + - make lint when: always artifacts: paths: diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..462303c --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +# Use bash shell with pipefail option enabled so that the return status of a +# piped command is the value of the last (rightmost) commnand to exit with a +# non-zero status. This lets us pipe output into tee but still exit on test +# failures. +SHELL = /bin/bash +.SHELLFLAGS = -o pipefail -c + +all: test lint + +# The following steps copy across useful output to this volume which can +# then be extracted to form the CI summary for the test procedure. +test: + + python setup.py test | tee ./build/setup_py_test.stdout; \ + mv coverage.xml ./build/reports/code-coverage.xml; + +# The following steps copy across useful output to this volume which can +# then be extracted to form the CI summary for the test procedure. +lint: + + # FIXME pylint needs to run twice since there is no way go gather the text and junit xml output at the same time + pip3 install pylint2junit; \ + pylint --output-format=parseable ska_python_skeleton | tee ./build/code_analysis.stdout; \ + pylint --output-format=pylint2junit.JunitReporter ska_python_skeleton > ./build/reports/linting.xml; + + +.PHONY: all test lint -- GitLab From 7e32739f1ee79ac2e3ceb6d9e1b0f627cfc68ada Mon Sep 17 00:00:00 2001 From: Adriaan de Beer Date: Thu, 12 Dec 2019 17:43:16 +0200 Subject: [PATCH 18/18] Removed commented code --- .gitlab-ci.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c24ed0b..7df5905 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,30 +51,6 @@ list_dependencies: paths: - public -# TODO: Not working properly at the moment - commented out temporarily -#code_quality: -# tags: -# - docker-executor -# image: docker:stable -# variables: -# DOCKER_DRIVER: overlay2 -# allow_failure: true -# services: -# - docker:stable-dind -# before_script: -# - ls -la -# script: -# - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') -# - docker run -# --env SOURCE_CODE="$PWD" -# --volume "$PWD":/code -# --volume /var/run/docker.sock:/var/run/docker.sock -# "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code -# - more gl-code-quality-report.json -# artifacts: -# paths: [gl-code-quality-report.json] - - linting: image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest tags: -- GitLab