Commit 8657778c authored by Elisabetta Giani's avatar Elisabetta Giani
Browse files

AT5-382: Work on CI pipeline configuration to generate the files to 

produce the metrics for the projects of the repository.
Not yet resolved the issue relatated to the combination of the
coverage generated by different projects hosted in the same repository.
Tried to use relative paths in vocerage (option [coverage:run] relative_files = True
in setup.cfg) with no success.
parent 4fa0308f

.gitlab-ci.yml

+43 −0
Original line number Diff line number Diff line
@@ -55,7 +55,46 @@ clean shell runner: 
    - docker-executor

  script:

    # Gitlab CI badges creation

    - pwd

    - ls -lR 

    - rm -rf build/*

#

# Pages

#

pages:

  tags:

    - docker-executor

  stage: pages

  variables:

    COMMON_PATH: csp-lmc-common/docker

    MIDCSP_PATH: csp-lmc-mid/docker

  script:

    - pwd

    - pip3 install junitparser

      #update coverage o the last release

    - pip3 install --upgrade coverage     

    - ls -lR

    # copy the content of packages folders in public

    # path to  all files is relative to the repository

    # that has been cloned during the build

    - cp -R $COMMON_PATH/build/ public

    - cp -R $MIDCSP_PATH/build/* public

    - ls -lR 

    - junitparser merge public/reports/csp-lmc-common-unit-tests.xml public/reports/csp-lmc-mid-unit-tests.xml public/reports/unit-tests.xml

    - ls -lR 

    - junitparser merge public/reports/csp-lmc-common-linting.xml public/reports/csp-lmc-mid-linting.xml public/reports/linting.xml

    - rm public/reports/csp-lmc-common-unit-tests.xml public/reports/csp-lmc-mid-unit-tests.xml

    - rm public/reports/csp-lmc-common-linting.xml public/reports/csp-lmc-mid-linting.xml

    - cp $COMMON_PATH/build/coverage.xml ./public/reports/code-coverage.xml

    #- cd public

    #- coverage combine csp-lmc-common_coverage csp-lmc-mid_coverage

    #- coverage xml -i

    # rename file because bagdges creation looks for code-coverage.xml file

    #- mv coverage.xml ./reports/code-coverage.xml

  artifacts:

    paths:

      - public 

    expire_in: 30 days



create ci metrics:

  stage: .post
@@ -64,6 +103,10 @@ create ci metrics: 
  tags:

    - docker-executor

  script:

    - ls -lR

    # copy public in build because ci-metrics scripts look for files in build

    - cp -R public build

    - ls -lR 

    # Gitlab CI badges creation: START

    - apt-get -y update

    - apt-get install -y curl --no-install-recommends

csp-lmc-common/.gitlab-ci.yml

+29 −14
Original line number Diff line number Diff line
@@ -13,30 +13,30 @@ 
    - engageska

  artifacts:

    paths:

      - ./$BUILD_PATH/dist/

      - ./$COMMON_BUILD_PATH/dist/



.build_python_dev_common:

  extends: .build_python_common

  dependencies: []

  script:

    - cd $BUILD_PATH

    - cd $COMMON_BUILD_PATH

    - python setup.py egg_info -b+dev.$CI_COMMIT_SHORT_SHA sdist bdist_wheel



.build_python_release_common:

  extends: .build_python_common

  script:

    - cd $BUILD_PATH

    - cd $COMMON_BUILD_PATH

    - python setup.py egg_info -b+$CI_COMMIT_SHORT_SHA sdist bdist_wheel



build:csp-lmc-common_pkg_dev: # Executed on a commit

  extends: .build_python_dev_common

  variables:

    BUILD_PATH: csp-lmc-common

    COMMON_BUILD_PATH: csp-lmc-common



build:csp-lmc-common_pkg:

  extends: .build_python_release_common

  variables:

    BUILD_PATH: csp-lmc-common

    COMMON_BUILD_PATH: csp-lmc-common

  only: [master]



#
@@ -51,10 +51,10 @@ build:csp-lmc-common_image: 
    - engageska

  stage: build_common

  script:

    - cd $BUILD_PATH      

    - cd $COMMON_BUILD_PATH      

    - make build

  variables:

    BUILD_PATH: csp-lmc-common/docker

    COMMON_BUILD_PATH: csp-lmc-common/docker



#  

# Test csp-lmc-common project
@@ -68,16 +68,31 @@ build:csp-lmc-common_image: 
    - docker-executor

  artifacts:

    paths:

      - build

      - ./$COMMON_BUILD_PATH/build/

  variables:

    BUILD_PATH: csp-lmc-common/docker

    COMMON_BUILD_PATH: csp-lmc-common/docker



test:csp-lmc-common:

  extends: .test_common

  stage: test_common

  script:

    - cd $BUILD_PATH  

    - cd $COMMON_BUILD_PATH  

    - make test

#

# linting stage

.linting_common:

  extends: .test_common

  stage: linting

  dependencies: []

  script:

    - apt-get -y update

    - apt-get install -y python3-pip python3-setuptools python3-wheel --no-install-recommends

    - cd $COMMON_BUILD_PATH  

    - make lint



linting:csp-lmc-common:

  extends: .linting_common

  dependencies: []



#

# Publish csp-lmc-common python package
@@ -108,7 +123,7 @@ test:csp-lmc-common: 
  extends: .publish_python_common

  only: [master]

  script:

    - cd $BUILD_PATH

    - cd $COMMON_BUILD_PATH

    - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true

    - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true
@@ -124,7 +139,7 @@ publish:csp-lmc-common: 
  dependencies:

    - build:csp-lmc-common_pkg

  variables:

    BUILD_PATH: csp-lmc-common

    COMMON_BUILD_PATH: csp-lmc-common



#

# Release csp-lmc-common image on nexus
@@ -136,11 +151,11 @@ release csp-lmc-common image: 
  dependencies:

    - build:csp-lmc-common_image

  script:

    - cd $BUILD_PATH      

    - cd $COMMON_BUILD_PATH      

    - make push

  when: manual

  only:

    refs:

      - master

  variables:

    BUILD_PATH: csp-lmc-common/docker
 
    COMMON_BUILD_PATH: csp-lmc-common/docker

csp-lmc-common/docker/test-harness/Makefile

+5 −5
Original line number Diff line number Diff line
@@ -21,15 +21,15 @@ test: 
	cd /app && python setup.py test | tee setup_py_test.stdout

	mkdir -p /build/reports && \

        if [ -d /build ]; then \

                mv /app/setup_py_test.stdout /build/csp_lmc_common_setup_test.stdout; \

                mv /app/htmlcov /build/csp_lmc_common_htmlcov; \

                mv /app/coverage.xml /build/csp_lmc_common_coverage.xml; \

                mv /app/setup_py_test.stdout /build/csp-lmc-common-setup-test.stdout; \

                mv /app/htmlcov /build/csp-lmc-common_htmlcov; \

                mv /app/coverage.xml /build; \

        fi;

lint:

	pip3 install pylint2junit; \

        mkdir -p /build/reports; \

        cd /app && pylint --output-format=parseable csp_lmc_common | tee /build/csp_lmc_common_code_analysis.stdout; \

        cd /app && pylint --output-format=pylint2junit.JunitReporter csp_lmc_common > /build/reports/cspl_lmc_common_linting.xml;

        cd /app && pylint --output-format=parseable csp_lmc_common | tee /build/csp_lmc_common-code-analysis.stdout; \

        cd /app && pylint --output-format=pylint2junit.JunitReporter csp_lmc_common > /build/reports/csp-lmc-common-linting.xml;



.PHONY: all test lint

csp-lmc-common/setup.cfg

+9 −2
Original line number Diff line number Diff line
@@ -7,8 +7,14 @@ test=pytest 


[coverage:run]

branch = True

#relative_files = True

source = csp-lmc-common



#data_file = csp-lmc-common_coverage

#debug = dataio,dataop

#[coverage:paths]

#source =

#    /app

#    ./csp-lmc/csp-lmc-common

[tool:pytest]

testpaths = tests

addopts = --verbose
@@ -18,7 +24,7 @@ addopts = --verbose 
          --cov-report=term 

          --cov-report=html 

          --cov-report=xml 

	  --junitxml=/build/reports/unit-tests.xml

	  --junitxml=/build/reports/csp-lmc-common-unit-tests.xml

console_output_style = progress

junit_family=legacy

filterwarnings =
@@ -30,3 +36,4 @@ source-dir = docs 
build-dir = docs/_build

all_files = 1

builder = html

csp-lmc-mid/.gitlab-ci.yml

+9 −29
Original line number Diff line number Diff line
@@ -25,7 +25,7 @@ 
  extends: .build_python

  script:

    - cd $BUILD_PATH

    - python setup.py sdist bdist_wheel

    - python setup.py egg_info -b+$CI_COMMIT_SHORT_SHA sdist bdist_wheel



build:csp-lmc-mid_pkg_dev: # Executed on a commit

  extends: .build_python_dev
@@ -83,7 +83,7 @@ build:csp-lmc-mid_image: 
    - docker-executor

  artifacts:

    paths:

      - build

      - ./$BUILD_PATH/build/

  variables:

    BUILD_PATH: csp-lmc-mid/docker
@@ -94,15 +94,21 @@ test:csp-lmc-mid: 
    - cd $BUILD_PATH  

    - make test



linting:

# linting stage

.linting:

  extends: .test

  stage: linting

  dependencies: []

  script:

    - apt-get -y update

    - apt-get install -y python3-pip python3-setuptools python3-wheel --no-install-recommends

    - cd $BUILD_PATH  

    - make lint



linting:csp-lmc-mid:

  extends: .linting

  dependencies: []



#

# Publish csp-lmc-mid python package

#
@@ -148,32 +154,6 @@ publish:csp-lmc-mid: 
  variables:

    BUILD_PATH: csp-lmc-mid



#

# Pages

#

pages:csp-lmc-mid:

  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest

  before_script:

    - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST

  tags:

    - docker-executor

  stage: pages

  dependencies: 

    - test:csp-lmc-mid

  only:

    - master

  script:

    - cd $BUILD_PATH

    - ls -la      

    - cp -R build public

    - mv build/htmlcov/* public  

  artifacts:

    paths:

      - public

    expire_in: 30 days

  variables:

    BUILD_PATH: csp-lmc-mid/docker



#

# Release csp-lmc-mid image on nexus

#