diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f8af45348a44ac6ccf599da65aac4f6bbc058b63..087474fa083acae5a63ef937b5414cdaf4ac696d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,6 +93,7 @@ pages: #- coverage xml -i # rename file because bagdges creation looks for code-coverage.xml file #- mv coverage.xml ./reports/code-coverage.xml + allow_failure: true artifacts: paths: - public diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml index 3535b76d2bdf7b5a5822e4f10124aa69909bfd3c..a2eeb6eaa2fa8ace8c79d226eb8d0d5eb7d7dfc8 100644 --- a/csp-lmc-common/.gitlab-ci.yml +++ b/csp-lmc-common/.gitlab-ci.yml @@ -6,7 +6,6 @@ .build_python_common: stage: build_common - dependencies: [] image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest tags: - k8srunner @@ -14,77 +13,73 @@ artifacts: paths: - ./$COMMON_BUILD_PATH/dist/ + variables: + COMMON_BUILD_PATH: csp-lmc-common -.build_python_dev_common: + +build:csp-lmc-common_pkg_dev: # Executed on a commit extends: .build_python_common - dependencies: [] script: - cd $COMMON_BUILD_PATH - python3 setup.py egg_info -b+dev.$CI_COMMIT_SHORT_SHA sdist bdist_wheel + except: + - master -.build_python_release_common: +build:csp-lmc-common_pkg: extends: .build_python_common script: - cd $COMMON_BUILD_PATH - python3 setup.py egg_info -b+$CI_COMMIT_SHORT_SHA sdist bdist_wheel - -build:csp-lmc-common_pkg_dev: # Executed on a commit - extends: .build_python_dev_common - variables: - COMMON_BUILD_PATH: csp-lmc-common - -build:csp-lmc-common_pkg: - extends: .build_python_release_common - variables: - COMMON_BUILD_PATH: csp-lmc-common - only: [master] + only: + - tags + - master # # Build csp-lmc-common image # -build:csp-lmc-common_image: - image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest - before_script: - - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST - tags: - - k8srunner - - engageska - stage: build_common - script: - - cd $COMMON_BUILD_PATH - - make build - variables: - COMMON_BUILD_PATH: csp-lmc-common/docker +#build:csp-lmc-common_image: +# image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest +# before_script: +# - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST +# tags: +# - k8srunner +# - engageska +# stage: build_common +# script: +# - cd $COMMON_BUILD_PATH +# - make build +# variables: +# COMMON_BUILD_PATH: csp-lmc-common/docker # # Test csp-lmc-common project # -.test_common: +test:csp-lmc-common: image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest + stage: test_common tags: - k8srunner - artifacts: - paths: - - ./$COMMON_BUILD_PATH/build/ - variables: - COMMON_BUILD_PATH: csp-lmc-common - -test:csp-lmc-common: - extends: .test_common - stage: test_common script: + - echo $CI_REGISTRY_IMAGE + - echo $CI_REGISTRY + - echo $CI_REGISTRY_USER - cd $COMMON_BUILD_PATH - pwd - python3 -m pip install -r requirements-tst.txt - - python3 -m pip install --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple -r requirements.txt . - - pytest tests |tee setup_py_test.stdout + - python3 -m pip install -r requirements.txt . + - python3 -m pytest tests | tee setup_py_test.stdout - pwd - mkdir -p build/reports - ls -alR - mv setup_py_test.stdout build/csp-lmc-common-setup-test.stdout - mv htmlcov build/csp-lmc-common_htmlcov - mv coverage.xml build + artifacts: + paths: + - ./$COMMON_BUILD_PATH/build/ + variables: + COMMON_BUILD_PATH: csp-lmc-common # # linting stage @@ -117,58 +112,59 @@ linting:csp-lmc-common: .publish_python_common: stage: publish_common image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest - variables: - TWINE_USERNAME: $TWINE_USERNAME - TWINE_PASSWORD: $TWINE_PASSWORD tags: - k8srunner before_script: - python3 -m pip install twine + variables: + COMMON_BUILD_PATH: csp-lmc-common -.publish_python_dev_common: - extends: .publish_python_common - except: [master] - script: - - cd $BUILD_PATH - - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true - -.publish_python_release_common: - extends: .publish_python_common - only: [master] - script: - - cd $COMMON_BUILD_PATH - - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true - - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true - +# upload the common package to the Gitlab Package Repository. +# Se the '--skip-existing' because if the CI/CD pipeline is re-run (without doing a commit) +# the package version does not change and the upload to the repository fails. publish:csp-lmc-common_dev: - extends: .publish_python_dev_common + extends: .publish_python_common dependencies: - build:csp-lmc-common_pkg_dev variables: BUILD_PATH: csp-lmc-common + script: + - cd $COMMON_BUILD_PATH + - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --skip-existing --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA) +# upload the package to the Nexus Repository only on project tagging. publish:csp-lmc-common: - extends: .publish_python_release_common + extends: .publish_python_common dependencies: - build:csp-lmc-common_pkg variables: - COMMON_BUILD_PATH: csp-lmc-common + TWINE_USERNAME: $TWINE_USERNAME + TWINE_PASSWORD: $TWINE_PASSWORD + only: + refs: + - tags + script: + - cd $COMMON_BUILD_PATH + - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true + - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true + + # # Release csp-lmc-common image on nexus # -release csp-lmc-common image: - tags: - - k8srunner - stage: release - dependencies: - - build:csp-lmc-common_image - script: - - cd $COMMON_BUILD_PATH - - make push - when: manual - only: - refs: - - master - variables: - COMMON_BUILD_PATH: csp-lmc-common/docker +#release csp-lmc-common image: +# tags: +# - k8srunner +# stage: release +# dependencies: +# - build:csp-lmc-common_image +# script: +# - cd $COMMON_BUILD_PATH +# - make push +# when: manual +# only: +# refs: +# - master +# variables: +# COMMON_BUILD_PATH: csp-lmc-common/docker diff --git a/csp-lmc-common/HISTORY b/csp-lmc-common/HISTORY index 1a8cca1444ced8092a4287d5367e6415419b7eb6..b7bd5ba43a5324ca6669573e6bfe40eb49c7cfdb 100644 --- a/csp-lmc-common/HISTORY +++ b/csp-lmc-common/HISTORY @@ -1,3 +1,6 @@ +0.6.12 + - support to ADR18/22 + 0.6.11 - fix bug in setup.py diff --git a/csp-lmc-common/csp_lmc_common/CspSubarray.py b/csp-lmc-common/csp_lmc_common/CspSubarray.py index 811da7e309e2e32fb40c4080e79a0a23d54a0974..3e0f3a8d3bd515f20e26db21fa90aabe86d8295d 100644 --- a/csp-lmc-common/csp_lmc_common/CspSubarray.py +++ b/csp-lmc-common/csp_lmc_common/CspSubarray.py @@ -43,6 +43,7 @@ from ska.base.control_model import HealthState, AdminMode, ObsState, ObsMode from .utils.cspcommons import CmdExecState from .utils.decorators import transaction_id from . import release +from .csp_manage_json import JsonConfiguration # PROTECTED REGION END # // CspSubarray.additionnal_import __all__ = ["CspSubarray", "main"] @@ -660,8 +661,6 @@ class CspSubarray(SKASubarray): self.logger.info("Configure ends with success!!") return target_device.configure_cmd_obj.succeeded() - - def validate_scan_configuration(self, argin): """ This method is overwritten in each CspSubarray instance. @@ -673,21 +672,19 @@ class CspSubarray(SKASubarray): target_device = self.target try: - json_dict = json.loads(argin) - if "id" not in json_dict: - msg = ("Command failed: No configuration ID in JSON object") - self.logger.error(msg) - tango.Except.throw_exception("Command failed", - msg, - "ConfigureScan execution", - tango.ErrSeverity.ERR) - self._configuration_id = json_dict["id"] + json_config = JsonConfiguration(argin, self.logger) + target_device._configuration_id = json_config.get_id() self.logger.info("Processing configuration {}".format(target_device._configuration_id)) - except json.JSONDecodeError as e: # argument not a valid JSON object + except Exception as e: # argument not a valid JSON object # this is a fatal error - msg = ("Scan configuration object is not a valid JSON object." - "Aborting configuration:{}".format(str(e))) + if isinstance(e, json.JSONDecodeError): + msg = ("Scan configuration object is not a valid JSON object." + "Aborting configuration:{}".format(str(e))) + elif isinstance(e, ValueError): + msg = "Command failed: No configuration ID in JSON object" + else: + msg = 'Unknown Error' self.logger.error(msg) tango.Except.throw_exception("Command failed", msg, diff --git a/csp-lmc-common/csp_lmc_common/csp_manage_json.py b/csp-lmc-common/csp_lmc_common/csp_manage_json.py new file mode 100644 index 0000000000000000000000000000000000000000..9d88f013b434eea43600d5a49a7a9f9c8c228097 --- /dev/null +++ b/csp-lmc-common/csp_lmc_common/csp_manage_json.py @@ -0,0 +1,85 @@ +import json +from ska_telmodel.csp.schema import validate_csp_config +from ska_telmodel.csp.version import validate_csp_config_version + +class JsonConfiguration: + """ + A set of methods to manage the json configuration inside CSP code + """ + def __init__(self, json_config:str, logger): + self.major_version = None + self.minor_version = None + self.logger = logger + self.config = json.loads(json_config) + + def detect_version(self): + """ + Detect the version of the json configuration + """ + #ADR-22 + if 'interface' in self.config.keys(): + interface = self.config['interface'] + if validate_csp_config_version(interface): + version_str = interface.split('/')[-1] + self.major_version , self.minor_version = map(int, version_str.split('.')) + else: + message = 'Wrong json schema URI!' + self.logger.error(message) + raise ValueError(message) + #ADR-18 + elif 'common' in self.config.keys(): + self.major_version, self.minor_version = (1,0) + #pre ADR-18 + else: + self.major_version, self.minor_version = (0,1) + self.version = (self.major_version, self.minor_version) + self.logger.info(f'Version is{self.major_version}{self.minor_version}') + + def get_section(self, name:str): + """ + Returns the correspondant section of json configuration. + """ + try: + return self.config[name] + except KeyError as e: + self.logger.error(e) + raise ValueError(e) from e + + def get_id(self): + """ + Return the json configuration id + """ + try: + common_sec = self.get_section('common') + config_id = common_sec['id'] + except ValueError as e: + if 'id' in self.config: + config_id = self.config['id'] + else: + raise ValueError(e) from e + return config_id + + def conversion_10_01(self): + """ + Converts the json from version 1.0 (ADR-18) to version 0.1. + Note: Current Mid.CBF supports 0.1 Json schema. + """ + common_dict = self.get_section('common') + cbf_dict = self.get_section('cbf') + config_converted = {**common_dict, **cbf_dict} + validate_csp_config(version=0, config=config_converted) + self.config = config_converted + + def build_json_cbf(self) -> str: + """ + Returns the json configuration dictionary to be passed to cbf. + If a version 0.2 is given, it converts to 0.1 + """ + self.detect_version() + if self.version == (0, 1): + pass + elif self.version == (1, 0): + self.conversion_10_01() + else: + raise ValueError(f'version{self.version}is not supported') + return json.dumps(self.config) diff --git a/csp-lmc-common/csp_lmc_common/release.py b/csp-lmc-common/csp_lmc_common/release.py index 66666d629bc26e30371dc22f1873921f45259ef6..28b5231ec64614b1691fca6d5da92bc5f0eb5b5a 100755 --- a/csp-lmc-common/csp_lmc_common/release.py +++ b/csp-lmc-common/csp_lmc_common/release.py @@ -10,7 +10,7 @@ """Release information for Python Package""" name = """csp-lmc-common""" -version = "0.6.11" +version = "0.6.12" version_info = version.split(".") description = """SKA CSP.LMC Common Software""" author = "INAF-OAA" diff --git a/csp-lmc-common/requirements.txt b/csp-lmc-common/requirements.txt index e302923ee06c4463cfba542ad40e4ee8640c142a..6f987de9eabf63d2cd96de9cbc38e02c5b853df2 100644 --- a/csp-lmc-common/requirements.txt +++ b/csp-lmc-common/requirements.txt @@ -1,5 +1,6 @@ numpy == 1.17.2 pytango >= 9.3.2 jsonschema >= 3.2.0 -lmcbaseclasses >= 0.7.2 +lmcbaseclasses >= 0.8.1 ska-log-transactions +ska-telescope-model==0.2.0 diff --git a/csp-lmc-common/setup.py b/csp-lmc-common/setup.py index aa15c8fd487e0b43876392bcfd7651eb6ee0082c..ca547d011160c02541594c360a21da9b6c607149 100644 --- a/csp-lmc-common/setup.py +++ b/csp-lmc-common/setup.py @@ -40,7 +40,7 @@ setup( 'pytango >9.3.1', 'future', 'lmcbaseclasses', - 'ska-log-transactions' + 'ska-telescope-model==0.2.0' ], dependency_links=[ 'https://nexus.engageska-portugal.pt/repository/pypi/simple' @@ -48,7 +48,7 @@ setup( setup_requires=[ 'pytest-runner', 'sphinx', - 'recommonmark' + 'recommonmark', ], test_suite="test", entry_points={'console_scripts':[ diff --git a/csp-lmc-common/tests/unit/json_config_test.py b/csp-lmc-common/tests/unit/json_config_test.py new file mode 100644 index 0000000000000000000000000000000000000000..b2dcf50101bb05adf0f5def0e189b646113b051d --- /dev/null +++ b/csp-lmc-common/tests/unit/json_config_test.py @@ -0,0 +1,188 @@ +import json +from csp_lmc_common.csp_manage_json import JsonConfiguration +import logging +import pytest + +LOGGER = logging.getLogger(__name__) + +json_string_test_ADR_22 = """ +{ + "interface": "https://schema.skatelescope.org/ska-csp-configure/1.0", + "common": { + "id": "sbi-mvp01-20200325-00001-science_A", + "frequencyBand": "1" + }, + "cbf": { + "fsp": [ + { + "fspID": 1, "functionMode": "CORR", "frequencySliceID": 1, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 0, + "outputLinkMap": [[0,0], [200,1]] + }, + { + "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 744, + "outputLinkMap": [[0,4], [200,5]] + } + ] + } +} +""" +json_string_test_ADR_18 = """ +{ + "common": { + + "id": "sbi-mvp01-20200325-00001-science_A", + "frequencyBand": "1" + }, + "cbf": { + "fsp": [ + { + "fspID": 1, "functionMode": "CORR", "frequencySliceID": 1, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 0, + "outputLinkMap": [[0,0], [200,1]] + }, + { + "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 744, + "outputLinkMap": [[0,4], [200,5]] + } + ] + } +} +""" +#function mode missing +json_string_test_ADR_18_invalid = """ +{ + "common": { + + "id": "sbi-mvp01-20200325-00001-science_A", + "frequencyBand": "1" + }, + "cbf": { + "fsp": [ + { + "fspID": 1, "frequencySliceID": 1, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 0, + "outputLinkMap": [[0,0], [200,1]] + }, + { + "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 744, + "outputLinkMap": [[0,4], [200,5]] + } + ] + } +} +""" +json_string_test_pre_ADR_18 = """ +{ + "id": "sbi-mvp01-20200325-00001-science_A", + "frequencyBand": "1", + "fsp": [ + { + "fspID": 1, "functionMode": "CORR", "frequencySliceID": 1, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 0, + "outputLinkMap": [[0,0], [200,1]] + }, + { + "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 744, + "outputLinkMap": [[0,4], [200,5]] + } + ] +} +""" + +json_string_invalid_URI = '{"interface": "invaliduri.com/0.1"}' + +class TestJsonConfiguration: + + def init_config(self): + self.json_config_ADR22 = JsonConfiguration(json_string_test_ADR_22, LOGGER) + self.json_config_ADR18 = JsonConfiguration(json_string_test_ADR_18, LOGGER) + self.json_config_pre_ADR18 = JsonConfiguration(json_string_test_pre_ADR_18, LOGGER) + + + def test_detect_version(self): + self.init_config() + self.json_config_ADR22.detect_version() + assert self.json_config_ADR22.major_version == 1 + assert self.json_config_ADR22.minor_version == 0 + + self.json_config_ADR18.detect_version() + assert self.json_config_ADR18.major_version == 1 + assert self.json_config_ADR18.minor_version == 0 + + self.json_config_pre_ADR18.detect_version() + assert self.json_config_pre_ADR18.major_version == 0 + assert self.json_config_pre_ADR18.minor_version == 1 + + def test_detect_version_with_invalid_uri(self): + with pytest.raises(ValueError, match= r'Wrong json schema URI!'): + JsonConfiguration(json_string_invalid_URI, LOGGER).detect_version() + + def test_get_section_when_section_is_present(self): + self.init_config() + assert isinstance(self.json_config_ADR18.get_section('cbf'), dict) + assert 'fsp' in self.json_config_ADR18.get_section('cbf').keys() + + assert isinstance(self.json_config_pre_ADR18.get_section('fsp'), list) + + def test_get_section_when_section_is_not_present(self): + self.init_config() + with pytest.raises(ValueError): + self.json_config_pre_ADR18.get_section('cbf') + + with pytest.raises(ValueError): + self.json_config_ADR18.get_section('fsp') + + def test_conversion_10_01(self): + self.init_config() + self.json_config_ADR18.conversion_10_01() + assert self.json_config_ADR18.config == self.json_config_pre_ADR18.config + + def test_conversion_10_01_invalid_schema(self): + with pytest.raises(ValueError): + JsonConfiguration(json_string_test_ADR_18_invalid, LOGGER).conversion_10_01() + + def test_build_json_cbf_version(self): + self.init_config() + assert self.json_config_ADR18.build_json_cbf() == self.json_config_pre_ADR18.build_json_cbf() + assert self.json_config_ADR22.build_json_cbf() == self.json_config_pre_ADR18.build_json_cbf() + + def test_build_json_cbf_version_not_supported(self): + self.init_config() + # version in interface is changed to 2.0 to force the failing of the test + self.json_config_ADR22.config['interface'] = "https://schema.skatelescope.org/ska-csp-configure/2.0" + with pytest.raises(ValueError, match=r'version.*is not supported'): + self.json_config_ADR22.build_json_cbf() + + def test_get_id(self): + self.init_config() + assert self.json_config_pre_ADR18.get_id() == "sbi-mvp01-20200325-00001-science_A" + assert self.json_config_ADR18.get_id() == "sbi-mvp01-20200325-00001-science_A" + + def test_get_id_NOT_PRESENT(self): + self.init_config() + self.json_config_pre_ADR18.config.pop('id') + json_config_with_no_id = json.dumps(self.json_config_pre_ADR18.config) + json_config_with_no_id = JsonConfiguration(json_config_with_no_id, LOGGER) + with pytest.raises(ValueError): + json_config_with_no_id.get_id() diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml index 5677fe877ea83c8cb4cc7a3534023be31c709dce..df4da27fab908481a666099b59e42782530db029 100644 --- a/csp-lmc-mid/.gitlab-ci.yml +++ b/csp-lmc-mid/.gitlab-ci.yml @@ -3,74 +3,50 @@ # # Build csp-lmc-mid python packages # - -.build_python: +build:csp-lmc-mid_pkg: stage: build image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest dependencies: [] tags: - k8srunner - artifacts: - paths: - - ./$BUILD_PATH/dist/ - -.build_python_dev: - extends: .build_python - dependencies: [] - script: - - cd $BUILD_PATH - - python3 setup.py egg_info -b+dev.$CI_COMMIT_SHORT_SHA sdist bdist_wheel - -.build_python_release: - extends: .build_python script: - cd $BUILD_PATH - python3 setup.py egg_info -b+$CI_COMMIT_SHORT_SHA sdist bdist_wheel - -build:csp-lmc-mid_pkg_dev: # Executed on a commit - extends: .build_python_dev variables: BUILD_PATH: csp-lmc-mid - -build:csp-lmc-mid_pkg: - extends: .build_python_release - variables: - BUILD_PATH: csp-lmc-mid - only: [master] + artifacts: + paths: + - ./$BUILD_PATH/dist/ + only: + - tags # -# Build and push csp-lmc-mid image +# Build and push csp-lmc-mid image on the gitlab Container Registry. +# Use the csp-lmc-common package stored in the gitlab Package Registry. +# The image is used to run the integration tests running in k8s environment. # + build:csp-lmc-mid_image: image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest before_script: - - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST + - cd $COMMON_PATH + - export COMMON_VER=$(python3 setup.py --version) + - echo $COMMON_VER + - cd .. tags: - k8srunner stage: build script: - cd $BUILD_PATH - - make build - - make push + - echo $IMG + - docker build -t $IMG --build-arg PROJECT_ID=$CI_PROJECT_ID --build-arg PKG_VER=$COMMON_VER --build-arg RELEASE="+dev.$CI_COMMIT_SHORT_SHA" -f Dockerfile.gitlab . + - echo -n "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY + - docker push "$IMG" variables: BUILD_PATH: csp-lmc-mid + COMMON_PATH: csp-lmc-common + IMG: "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA" -# test:csp-lmc-mid: -# stage: test -# image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest -# before_script: -# - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST -# - *install_pip -# tags: -# - k8srunner -# script: -# - echo $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA) -# - pip3 install -U $(ls -d ./csp-lmc-mid/dist/*.whl | grep $CI_COMMIT_SHORT_SHA) --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple -# - cd csp-lmc-mid/docker -# - make test -# artifacts: -# paths: -# - build # # Test csp-lmc-mid project @@ -108,6 +84,10 @@ integration:csp-lmc-mid: script: - export KUBE_NAMESPACE="ci-csp-proto-$CI_COMMIT_SHORT_SHA" - cd $BUILD_PATH + - make k8s + - make install-chart + - make template-chart + - make wait - make test environment: name: test @@ -144,9 +124,9 @@ linting:csp-lmc-mid: # # Publish csp-lmc-mid python package +# This stage runs only for tags. # - -.publish_python: +publish:csp-lmc-mid: stage: publish image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest variables: @@ -156,36 +136,17 @@ linting:csp-lmc-mid: - k8srunner before_script: - pip install twine - -.publish_python_dev: - extends: .publish_python - except: [master] - script: - - cd $BUILD_PATH - - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true - when: manual - -.publish_python_release: - extends: .publish_python - only: [master] + dependencies: + - build:csp-lmc-mid_pkg script: - cd $BUILD_PATH - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true - -publish:csp-lmc-mid_dev: - extends: .publish_python_dev - dependencies: - - build:csp-lmc-mid_pkg_dev - variables: - BUILD_PATH: csp-lmc-mid - -publish:csp-lmc-mid: - extends: .publish_python_release - dependencies: - - build:csp-lmc-mid_pkg variables: BUILD_PATH: csp-lmc-mid + only: + refs: + - tags publish_chart: variables: @@ -203,10 +164,14 @@ publish_chart: - curl -s https://gitlab.com/ska-telescope/stupid/raw/master/scripts/publish-charts.sh | bash variables: BUILD_PATH: csp-lmc-mid + # -# Release csp-lmc-mid image on nexus +# Build and release csp-lmc-mid image on nexus +# This stage runs only for tags. # release csp-lmc-mid image: + before_script: + - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST tags: - k8srunner stage: release @@ -214,10 +179,11 @@ release csp-lmc-mid image: - build:csp-lmc-mid_image script: - cd $BUILD_PATH + - make build - make push - when: manual + when: on_success only: refs: - - master + - tags variables: BUILD_PATH: csp-lmc-mid diff --git a/csp-lmc-mid/.make/k8s.mk b/csp-lmc-mid/.make/k8s.mk index 2d8834390ebfcfa93a3ca00b905ba88e58bdcd36..c3eb95b118daa4e4f96b7b4cf3d836ab53a9ebc3 100644 --- a/csp-lmc-mid/.make/k8s.mk +++ b/csp-lmc-mid/.make/k8s.mk @@ -12,6 +12,13 @@ CI_ENVIRONMENT_SLUG ?= mid-csp # in release.mk it's defined the value of the variable IMAGE_TAG SET_IMAGE_TAG ?= --set mid-csp.midcsplmc.image.tag=$(IMAGE_TAG) +ifneq ($(CI_JOB_ID),) +CI_PROJECT_IMAGE := +SET_IMAGE_TAG = --set mid-csp.midcsplmc.image.registry=$(CI_REGISTRY)/ska-telescope \ + --set mid-csp.midcsplmc.image.image=csp-lmc \ + --set mid-csp.midcsplmc.image.tag=$(CI_COMMIT_SHORT_SHA) +IMAGE_TO_TEST = $(CI_REGISTRY_IMAGE):$(CI_COMMIT_SHORT_SHA) +endif .DEFAULT_GOAL := help k8s: ## Which kubernetes are we connected to @@ -207,7 +214,7 @@ kubeconfig: ## export current KUBECONFIG as base64 ready for KUBE_CONFIG_BASE64 # base64 payload is given a boundary "~~~~BOUNDARY~~~~" and extracted using perl # clean up the run to completion container # exit the saved status -test: install-chart wait ## test the application on K8s +test: ## test the application on K8s $(call k8s_test,test); \ status=$$?; \ rm -rf build; \ diff --git a/csp-lmc-mid/.make/release.mk b/csp-lmc-mid/.make/release.mk index e1597a486126937cd4272e94fa96b7aa65edc80b..2ac90c453b1ea6e0414e6e172c11f1ad7505e3e7 100644 --- a/csp-lmc-mid/.make/release.mk +++ b/csp-lmc-mid/.make/release.mk @@ -42,7 +42,7 @@ BASE_RELEASE=$(shell . $(RELEASE_SUPPORT) ;getRelease) BASE_VERSION=$(shell . $(RELEASE_SUPPORT) ;getBaseVersion) ifeq ($(BRANCH_NAME), master) -IMAGE_TAG ?= $(BASE_RELEASE) +IMAGE_TAG ?= $(BASE_VERSION) else IMAGE_TAG ?= $(BASE_VERSION)-dev endif diff --git a/csp-lmc-mid/.release b/csp-lmc-mid/.release index f5a4952d08c8729592189000f331068fbf70666d..8ad0f4b49581b0389b5d6e6d27782fd0beadc8a1 100644 --- a/csp-lmc-mid/.release +++ b/csp-lmc-mid/.release @@ -9,7 +9,7 @@ """Release information for Python Package""" name = """MID CSP.LMC""" -version = "0.6.9" +version = "0.6.10" version_info = version.split(".") description = """SKA MID CSP.LMC Classes""" author = "E.G" @@ -18,5 +18,5 @@ license = """BSD-3-Clause""" url = """www.tango-controls.org""" copyright = """""" -release=0.6.9 -tag=mid-csp-lmc-0.6.9 +release=0.6.10 +tag=mid-csp-lmc-0.6.10 diff --git a/csp-lmc-mid/Dockerfile b/csp-lmc-mid/Dockerfile index 2456a5e1e6f5bf0461d1e867ef309747b2f2be11..d2b1bea00885f2499e28bb60802ae4396bbd4ffb 100644 --- a/csp-lmc-mid/Dockerfile +++ b/csp-lmc-mid/Dockerfile @@ -5,6 +5,6 @@ FROM nexus.engageska-portugal.pt/ska-docker/ska-python-runtime:9.3.2 AS runtime RUN ipython profile create ENV PATH=/home/tango/.local/bin:$PATH -RUN python3 -m pip install . --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple +RUN python3 -m pip install -r requirements.txt . CMD ["/venv/bin/python","/app/csp_lmc_mid/MidCspMaster.py"] diff --git a/csp-lmc-mid/Dockerfile.gitlab b/csp-lmc-mid/Dockerfile.gitlab new file mode 100644 index 0000000000000000000000000000000000000000..b17ac41a9c5919ca7d5d0b7b2b8237ad9a8eb91c --- /dev/null +++ b/csp-lmc-mid/Dockerfile.gitlab @@ -0,0 +1,23 @@ +# Dockerfile used during CI/CD pipeline execution. +# The docker image is built using the csp-lmc-common python +# package uploaded to the gitlab Package Repository. +# PROJECT_ID: is the csp-lmc project ID under Gitlab: the value is passed as argument +# during the image build stage. +FROM nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:9.3.2 AS buildenv +FROM nexus.engageska-portugal.pt/ska-docker/ska-python-runtime:9.3.2 AS runtime + +# create ipython profile to so that itango doesn't fail if ipython hasn't run yet +RUN ipython profile create + +ENV PATH=/home/tango/.local/bin:$PATH +ARG PROJECT_ID +ARG RELEASE +ARG PKG_VER +# install all ska packages from nexus repository before installing the dev +# csp-lmc-common package. In this way the csp-lmc-common dependencies are already +# in place and the package can be installed from the gitlab repo without errors. +RUN python3 -m pip install -r requirements-gitlab.txt . +#install +RUN python3 -m pip install -U --index-url https://gitlab.com/api/v4/projects/$PROJECT_ID/packages/pypi/simple csp-lmc-common=="$PKG_VER$RELEASE" + +CMD ["/venv/bin/python","/app/csp_lmc_mid/MidCspMaster.py"] diff --git a/csp-lmc-mid/HISTORY b/csp-lmc-mid/HISTORY index 4de4f759ca2cb413ffae8f26b27b73b91aa67d4f..db304070b0d0c8a07f959638aa9750c1bd9c0e8b 100644 --- a/csp-lmc-mid/HISTORY +++ b/csp-lmc-mid/HISTORY @@ -1,3 +1,8 @@ +0.6.10 +- use csp-lmc-common 0.6.12 +- support to ADR-18/22 +- use ska-telescope-model 0.2.0 + 0.6.9: - use csp-lmc-common 0.6.9 - fix bug in receptor class diff --git a/csp-lmc-mid/charts/mid-csp-umbrella/Chart.yaml b/csp-lmc-mid/charts/mid-csp-umbrella/Chart.yaml index f99d2365d19d9b72cd8c6fe8d16759c57a51bbcc..7ee78858c6797d2973c3c1c323c18bd43bcf7bd2 100644 --- a/csp-lmc-mid/charts/mid-csp-umbrella/Chart.yaml +++ b/csp-lmc-mid/charts/mid-csp-umbrella/Chart.yaml @@ -6,7 +6,7 @@ appVersion: 0.6.9 icon: https://www.skatelescope.org/wp-content/uploads/2016/07/09545_NEW_LOGO_2014.png dependencies: - name: tango-base - version: 0.2.7 + version: 0.2.12 repository: https://nexus.engageska-portugal.pt/repository/helm-chart - name: mid-cbf version: 0.1.1 @@ -15,5 +15,5 @@ dependencies: version: 0.1.1 repository: https://nexus.engageska-portugal.pt/repository/helm-chart - name: mid-csp - version: 0.1.0 + version: 0.1.1 repository: "file://../mid-csp" diff --git a/csp-lmc-mid/charts/mid-csp/Chart.yaml b/csp-lmc-mid/charts/mid-csp/Chart.yaml index ccd8ef2dcd0ed0c07b8e7944119017379413e5f0..63f811907aa83cec2bb451bfc9b24cd4c65dfa37 100644 --- a/csp-lmc-mid/charts/mid-csp/Chart.yaml +++ b/csp-lmc-mid/charts/mid-csp/Chart.yaml @@ -1,15 +1,15 @@ apiVersion: v2 name: mid-csp description: A Helm chart for deploying the Mid_CSP.LMC devices on Kubernetes -version: 0.1.0 -appVersion: "0.6.9" +version: 0.1.1 +appVersion: "0.6.10" icon: https://www.skatelescope.org/wp-content/uploads/2016/07/09545_NEW_LOGO_2014.png dependencies: - name: tango-util - version: 0.2.4 + version: 0.2.8 repository: https://nexus.engageska-portugal.pt/repository/helm-chart - name: tango-base - version: 0.2.7 + version: 0.2.12 repository: https://nexus.engageska-portugal.pt/repository/helm-chart condition: tango-base.enabled,global.sub-system.tango-base.enabled diff --git a/csp-lmc-mid/charts/mid-csp/values.yaml b/csp-lmc-mid/charts/mid-csp/values.yaml index 42b5db9ac1c188d20e486bd2553c827747535f27..2da0693ee20a8ecbb64a87aa20a6baa3b22672a3 100644 --- a/csp-lmc-mid/charts/mid-csp/values.yaml +++ b/csp-lmc-mid/charts/mid-csp/values.yaml @@ -37,7 +37,7 @@ midcsplmc: image: registry: nexus.engageska-portugal.pt/ska-docker image: mid-csp-lmc - tag: 0.6.9 + tag: 0.6.10 pullPolicy: IfNotPresent resources: diff --git a/csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py b/csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py index 21e65a4fb740be6229db438a8725024d7ad3784e..d517ba4f62797c0dafe9dae439cd671f800e285a 100644 --- a/csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py +++ b/csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py @@ -47,6 +47,7 @@ from csp_lmc_common.utils.cspcommons import CmdExecState #from csp_lmc_common.CspSubarray import CspSubarray, CspSubarrayStateModel from csp_lmc_common.CspSubarray import CspSubarray from csp_lmc_mid.receptors import Receptors +from csp_lmc_common.csp_manage_json import JsonConfiguration # PROTECTED REGION END # // MidCspSubarrayBase.additionnal_import __all__ = ["MidCspSubarrayBase", "main"] @@ -392,7 +393,8 @@ class MidCspSubarrayBase(CspSubarray): self.logger.debug("Validate scan configuration for MID CSP") #self._sc_subarray_obs_mode[self.CbfSubarray] = ObsMode.IMAGING #self._sc_scan_configuration[self.CbfSubarray] = json_config["cbf"] - device._sc_subarray_scan_configuration[device.CbfSubarray] = argin + json_config = JsonConfiguration(argin, self.logger) + device._sc_subarray_scan_configuration[device.CbfSubarray] = json_config.build_json_cbf() self.logger.debug("Validate scan: {}".format(device._sc_subarray_assigned_fqdn)) def __len__(self): diff --git a/csp-lmc-mid/csp_lmc_mid/release.py b/csp-lmc-mid/csp_lmc_mid/release.py index 102b81e728d3b44ff5fed02ca8176eaef992c003..1302a246d00235cb3a3b270210feeb9a2e789689 100755 --- a/csp-lmc-mid/csp_lmc_mid/release.py +++ b/csp-lmc-mid/csp_lmc_mid/release.py @@ -10,7 +10,7 @@ """Release information for Python Package""" name = """mid-csp-lmc""" -version = "0.6.9" +version = "0.6.10" version_info = version.split(".") description = """SKA MID CSP.LMC""" author = "INAF-OAA" diff --git a/csp-lmc-mid/requirements-gitlab.txt b/csp-lmc-mid/requirements-gitlab.txt new file mode 100644 index 0000000000000000000000000000000000000000..6bcd9f2a1f12791c53baab66eacde88d6dcf1970 --- /dev/null +++ b/csp-lmc-mid/requirements-gitlab.txt @@ -0,0 +1,7 @@ +numpy == 1.17.2 +pytango >= 9.3.2 +jsonschema >= 3.2.0 +lmcbaseclasses >=0.8.1 +ska-log-transactions +ska-telescope-model==0.2.0 + diff --git a/csp-lmc-mid/requirements.txt b/csp-lmc-mid/requirements.txt index 5a67db3c1938b40a61f402bfa8f89dedc2f47420..f8079945ab10c4683ee2269e5b07e1adb20dfd74 100644 --- a/csp-lmc-mid/requirements.txt +++ b/csp-lmc-mid/requirements.txt @@ -1,5 +1,8 @@ numpy == 1.17.2 pytango >= 9.3.2 jsonschema >= 3.2.0 -lmcbaseclasses >= 0.6.4 -csp-lmc-common >= 0.6.8 +lmcbaseclasses >= 0.8.0 +csp-lmc-common >= 0.6.12 +ska-log-transactions +ska-telescope-model==0.2.0 + diff --git a/csp-lmc-mid/setup.py b/csp-lmc-mid/setup.py index 14177cd6edb1b9175dc802f3d711fddf1c4dacac..09ee7edc6528ec1e85d279ba4a081671f2d0dea0 100644 --- a/csp-lmc-mid/setup.py +++ b/csp-lmc-mid/setup.py @@ -37,8 +37,7 @@ setup( include_package_data=True, install_requires = [ 'pytango >=9.3.1', - 'future', - 'csp-lmc-common>=0.6.9' + 'future' ], setup_requires=[ 'pytest-runner', diff --git a/csp-lmc-mid/tests/integration/MidCspSubarray_test.py b/csp-lmc-mid/tests/integration/MidCspSubarray_test.py index df272434decc3738b689c8d6bab91a8101ff385d..8bd86601a0e0010107d401cd60500837a1db0a86 100755 --- a/csp-lmc-mid/tests/integration/MidCspSubarray_test.py +++ b/csp-lmc-mid/tests/integration/MidCspSubarray_test.py @@ -35,7 +35,7 @@ def prepare_configuration_string(filename="test_ConfigureScan_ADR4.json"): """Create the config string for CSP-CBF""" try: file_to_load = file_path + '/../test_data/' + filename - LOGGER.debug(f" Reading configuration from {file_to_load}") + LOGGER.info(f" Reading configuration from {file_to_load}") json_file = open(file_to_load) configuration_string = json_file.read().replace("\n", "") return configuration_string @@ -406,6 +406,27 @@ class TestCspSubarray(TestBase): #stored_id = self.midcsp_subarray01.configurationID #assert stored_id == configID + @pytest.mark.csp_k8s + def test_send_configure_WITH_ADR22_json(self): + """ + Configure the CSP Subarray with a JSon string including + the new ADR4 fields. + """ + self._setup_subarray() + self._assign_receptors() + state = self.midcsp_subarray01.State() + obs_state = self.midcsp_subarray01.obsState + LOGGER.info("CSP Subarray State before exercise :{}-{}".format(state, ObsState(obs_state).name)) + # exercise the device + LOGGER.info(f"Configuring CSP subarray01") + configuration_string = prepare_configuration_string("test_ConfigureScan_ADR22.json") + (result_code, msg) = self.midcsp_subarray01.Configure(configuration_string) + # check + prober_subarray_obstate = Probe(self.midcsp_subarray01, 'obsState', ObsState.READY, + f"Wrong CSP Subarray obsState {self.midcsp_subarray01.obsState}") + Poller(5, 0.2).check(prober_subarray_obstate) + obs_state = self.midcsp_subarray01.obsState + @pytest.mark.csp_k8s def test_start_end_scan(self): """ diff --git a/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json b/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json new file mode 100644 index 0000000000000000000000000000000000000000..a326dd87a5f1556dbc09be746a3f067a99aeaa33 --- /dev/null +++ b/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json @@ -0,0 +1,29 @@ +{ + "interface": "https://schema.skatelescope.org/ska-csp-configure/1.0", + "common": { + "id": "sbi-mvp01-20200326-00001-science_B", + "frequencyBand": "3" + }, + "cbf": { + "delayModelSubscriptionPoint": "ska_mid/tm_leaf_node/csp_subarray_01/delayModel", + "fsp": [ + { + "fspID": 1, "functionMode": "CORR", "frequencySliceID": 1, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 0, + "outputLinkMap": [[0,0], [200,1]], + "outputHost": [[0, "192.168.0.1"], [8184, "192.168.0.2"]], + "outputMac": [[0, "06-00-00-00-00-01"]], + "outputPort": [[0, 9000, 1], [8184, 9000, 1]] + }, + { + "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2, + "integrationTime": 1400, "corrBandwidth": 0, + "channelAveragingMap": [[0,2], [744,0]], + "fspChannelOffset": 744, + "outputLinkMap": [[0,4], [200,5]] + } + ] + } +} diff --git a/csp-lmc-mid/tests/unit/midcspsubarray_unit_test.py b/csp-lmc-mid/tests/unit/midcspsubarray_unit_test.py index 5867030025464b084fe042acf07b9b3a664c78bb..1f0b9efb18a8ee071179fe43e8f7ed5e727c2033 100644 --- a/csp-lmc-mid/tests/unit/midcspsubarray_unit_test.py +++ b/csp-lmc-mid/tests/unit/midcspsubarray_unit_test.py @@ -372,7 +372,7 @@ def test_midcspsubarray_obsstate_AFTER_configure(): tango_context.device.AddReceptors(receptor_list) #assert tango_context.device.obsState == ObsState.IDLE cbf_subarray_device_proxy_mock.configureDelay.return_value = 10 - configuration_string = load_json_file("test_ConfigureScan_ADR4.json") + configuration_string = load_json_file("test_ConfigureScan_ADR22.json") tango_context.device.Configure(configuration_string) #assert tango_context.device.obsState == ObsState.CONFIGURING dummy_event = create_dummy_event(cbf_subarray_fqdn, 'obsstate',ObsState.READY)