AT5-259: GitLab CI/CD pipelines configuration files.

# GitLab CI in conjunction with GitLab Runner can use Docker Engine to test and build any application.

# Docker, when used with GitLab CI, runs each job in a separate and isolated container using the predefined image that is set up in .gitlab-ci.yml.

# In this case we use the latest python docker image to build and test this project.

image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest

variables:

  DOCKER_DRIVER: overlay2

#image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest

services:

  - docker:dind

#services:

#   - docker:dind

# TODO: add file for LOW project

include:

  - local: csp-lmc-common/.gitlab-ci.yml

  - local: csp-lmc-mid/.gitlab-ci.yml

# cache is used to specify a list of files and directories which should be cached between jobs. You can only use paths that are within the project workspace.

# If cache is defined outside the scope of jobs, it means it is set globally and all jobs will use that definition

# before_script is used to define the command that should be run before all jobs, including deploy jobs, but after the restoration of artifacts.

# This can be an array or a multi-line string.

before_script:

  - docker login --username $DOCKER_REGISTRY_USERNAME --password $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST

#before_script:

#  - docker login --username $DOCKER_REGISTRY_USERNAME --password $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST

# The YAML file defines a set of jobs with constraints stating when they should be run.

# You can specify an unlimited number of jobs which are defined as top-level elements with an arbitrary name and always have to contain at least the script clause.

# In this case we have only the test job which produce an artifacts (it must be placed into a directory called "public")

# It is also specified that only the master branch will be subject of this job.

# stages:

# build:        build the docker images for csp-lmc-mid and csp-lmc-low

# test:         execute tests on csp-lmc-common, csp-lmc-mid, csp-lmc-low

# package:      create the python packages for common, mid and low

# publish:      publish on nexus the mid and low images and the  common python package

# pages:        create the badges

stages:

  - build

  - test

  - package

  - publish

  - pages

  - release

test:

  tags:

    - docker-executor

  stage: test

  script:

    - cd ./csp-lmc-mid/docker

    - make build

    - make test

    - make lint

  artifacts:

    paths:

      - build

.install_pip: &install_pip |-

  apt-get -y update && apt-get install -yq curl python3-distutils

  curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py

publish image:

clean shell runner:

  stage: .pre

  tags:

    - docker-executor

  stage: publish

  script:

    - make push

  when: manual

    # Gitlab CI badges creation

    - rm -rf build/*

pages:

  stage: pages

  dependencies: 

     - test

  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest

  before_script:

  - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST

  tags:

    - docker-executor

  stage: pages

  only:

    - master

  script:

   - mkdir -p .public/reports  

   - cp build/csp*.* .public

   - cp -r build/csp_master_htmlcov/* .public

   - cp -r build/csp_subarray_htmlcov/* .public

   - cp temp/ci-metrics-mock.json ci-metrics.json

   - cp build/lint_output.xml .public/reports

   - cp build/csp_master_coverage.xml .public/reports

   - cp build/csp_subarray_coverage.xml .public/reports

   - mv .public public

   - curl -s https://gitlab.com/ska-telescope/ci-metrics-utilities/raw/master/scripts/ci-badges-func.sh | sh

   - cp -R build public

   - mv build/htmlcov/* public

  artifacts:

    paths:

      - public

    expire_in: 30 days

create ci metrics:

  stage: .post

  image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest

  when: always

  tags:

    - docker-executor

  script:

    # Gitlab CI badges creation: START

    - apt-get -y update

    - apt-get install -y curl --no-install-recommends

    - curl -s https://gitlab.com/ska-telescope/ci-metrics-utilities/raw/master/scripts/ci-badges-func.sh | sh

    # Gitlab CI badges creation: END

  artifacts:

    paths:

      - ./build

# Local Gitlab-CI file for csp-lmc-mid project

# 

# Build csp-lmc-common python package

# 

.build_python:

  stage: build

  dependencies: []

  tags:

    - docker-executor

  artifacts:

    paths:

      - ./$BUILD_PATH/dist/

.build_python_dev:

  extends: .build_python

  dependencies: []

  script:

    - cd $BUILD_PATH

    - python setup.py egg_info -b+dev.$CI_COMMIT_SHORT_SHA sdist bdist_wheel

.build_python_release:

  extends: .build_python

  script:

    - cd $BUILD_PATH

    - python setup.py sdist bdist_wheel

build:csp-lmc-common_pkg_dev: # Executed on a commit

  extends: .build_python_dev

  variables:

    BUILD_PATH: csp-lmc-common

build:csp-lmc-common_pkg:

  extends: .build_python_release

  variables:

    BUILD_PATH: csp-lmc-mid

  only: [master]

#

# Build csp-lmc-common image 

#

 build:csp-lmc-common_image:

  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest

  before_script:

  - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST

  tags:

    - docker-executor

  stage: build

  script:

    - cd $BUILD_PATH      

    - make build

  variables:

    BUILD_PATH: csp-lmc-common/docker

#

# Publish csp-lmc-mid python package

#

.publish_python:

  stage: publish

  variables:

    TWINE_USERNAME: $TWINE_USERNAME

    TWINE_PASSWORD: $TWINE_PASSWORD

  tags:

    - docker-executor

  before_script:

    - pip install twine

.publish_python_dev:

  extends: .publish_python

  except: [master]

  script:

    - cd $BUILD_PATH

    - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true

.publish_python_release:

  extends: .publish_python

  only: [master]

  script:

    - cd $BUILD_PATH

    - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true

    - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true

publish:csp-lmc-mid_dev:

  extends: .publish_python_dev

  dependencies:

    - build:csp-lmc-common_pkg_dev

  variables:

    BUILD_PATH: csp-lmc-common

publish:csp-lmc-mid:

  extends: .publish_python_release

  dependencies:

    - build:csp-lmc-common_pkg

  variables:

    BUILD_PATH: csp-lmc-common

#

# Release csp-lmc-mid image on nexus

#

release csp-lmc-mid image:

  tags:

    - docker-executor

  stage: release

  dependencies:

    - build:csp-lmc-common_image

  script:

    - cd $BUILD_PATH      

    - make push

  when: manual

  only:

    refs:

      - master

  variables:

    BUILD_PATH: csp-lmc-common/docker

# Local Gitlab-CI file for csp-lmc-mid project

# 

# Build csp-lmc-mid python packages

# 

.build_python:

  stage: build

  dependencies: []

  tags:

    - docker-executor

  artifacts:

    paths:

      - ./$BUILD_PATH/dist/

.build_python_dev:

  extends: .build_python

  dependencies: []

  script:

    - cd $BUILD_PATH

    - python setup.py egg_info -b+dev.$CI_COMMIT_SHORT_SHA sdist bdist_wheel

.build_python_release:

  extends: .build_python

  script:

    - cd $BUILD_PATH

    - python setup.py sdist bdist_wheel

build:csp-lmc-mid_pkg_dev: # Executed on a commit

  extends: .build_python_dev

  variables:

    BUILD_PATH: csp-lmc-mid

build:csp-lmc-mid_pkg:

  extends: .build_python_release

  variables:

    BUILD_PATH: csp-lmc-mid

  only: [master]

#

# Build csp-lmc-mid image 

#

 build:csp-lmc-mid_image:

  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest

  before_script:

  - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST

  tags:

    - docker-executor

  stage: build

  script:

    - cd $BUILD_PATH      

    - make build

  variables:

    BUILD_PATH: csp-lmc-mid/docker

# test:csp-lmc-mid:

#  stage: test

#  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest

#  before_script:

#      - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST

#      - *install_pip

#  tags:

#    - docker-executor

#  script:

#    - echo $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)

#    - pip3 install -U $(ls -d ./csp-lmc-mid/dist/*.whl | grep $CI_COMMIT_SHORT_SHA) --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple

#    - cd csp-lmc-mid/docker

#    - make test

#  artifacts:

#    paths:

#      - build

#  

# Test csp-lmc-mid project

#

.test:

  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest

  before_script:

    - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST

  tags:

    - docker-executor

  artifacts:

    paths:

      - build

  variables:

    BUILD_PATH: csp-lmc-mid/docker

test:csp-lmc-mid:

  extends: .test

  stage: test

  script:

    - cd $BUILD_PATH  

    - make test

linting:

  extends: .test

  stage: linting

  script:

    - apt-get -y update

    - apt-get install -y python3-pip python3-setuptools python3-wheel --no-install-recommends

    - cd $BUILD_PATH  

    - make lint

#

# Publish csp-lmc-mid python package

#

.publish_python:

  stage: publish

  variables:

    TWINE_USERNAME: $TWINE_USERNAME

    TWINE_PASSWORD: $TWINE_PASSWORD

  tags:

    - docker-executor

  before_script:

    - pip install twine

.publish_python_dev:

  extends: .publish_python

  except: [master]

  script:

    - cd $BUILD_PATH

    - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true

.publish_python_release:

  extends: .publish_python

  only: [master]

  script:

    - cd $BUILD_PATH

    - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true

    - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true

publish:csp-lmc-mid_dev:

  extends: .publish_python_dev

  dependencies:

    - build:csp-lmc-mid_pkg_dev

  variables:

    BUILD_PATH: csp-lmc-mid

publish:csp-lmc-mid:

  extends: .publish_python_release

  dependencies:

    - build:csp-lmc-mid_pkg

  variables:

    BUILD_PATH: csp-lmc-mid

#

# Release csp-lmc-mid image on nexus

#

release csp-lmc-mid image:

  tags:

    - docker-executor

  stage: release

  dependencies:

    - build:csp-lmc-mid_image

  script:

    - cd $BUILD_PATH      

    - make push

  when: manual

  only:

    refs:

      - master

  variables:

    BUILD_PATH: csp-lmc-mid/docker