Merge branch 'CT-220' into 'master' (72f3a2cd) · Commits · Gianluca Marotta / csp-lmc-old

.gitlab-ci.yml

+1 −0

Original line number	Diff line number	Diff line
		@@ -93,6 +93,7 @@ pages:
		#- coverage xml -i
		# rename file because bagdges creation looks for code-coverage.xml file
		#- mv coverage.xml ./reports/code-coverage.xml
		allow_failure: true
		artifacts:
		paths:
		- public

csp-lmc-common/.gitlab-ci.yml

+72 −76

Original line number	Diff line number	Diff line
		@@ -6,7 +6,6 @@

		.build_python_common:
		stage: build_common
		dependencies: []
		image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
		tags:
		- k8srunner
		@@ -14,77 +13,73 @@
		artifacts:
		paths:
		- ./$COMMON_BUILD_PATH/dist/
		variables:
		COMMON_BUILD_PATH: csp-lmc-common

		.build_python_dev_common:

		build:csp-lmc-common_pkg_dev: # Executed on a commit
		extends: .build_python_common
		dependencies: []
		script:
		- cd $COMMON_BUILD_PATH
		- python3 setup.py egg_info -b+dev.$CI_COMMIT_SHORT_SHA sdist bdist_wheel
		except:
		- master

		.build_python_release_common:
		build:csp-lmc-common_pkg:
		extends: .build_python_common
		script:
		- cd $COMMON_BUILD_PATH
		- python3 setup.py egg_info -b+$CI_COMMIT_SHORT_SHA sdist bdist_wheel

		build:csp-lmc-common_pkg_dev: # Executed on a commit
		extends: .build_python_dev_common
		variables:
		COMMON_BUILD_PATH: csp-lmc-common

		build:csp-lmc-common_pkg:
		extends: .build_python_release_common
		variables:
		COMMON_BUILD_PATH: csp-lmc-common
		only: [master]
		only:
		- tags
		- master

		#
		# Build csp-lmc-common image
		#
		build:csp-lmc-common_image:
		image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
		before_script:
		- docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
		tags:
		- k8srunner
		- engageska
		stage: build_common
		script:
		- cd $COMMON_BUILD_PATH
		- make build
		variables:
		COMMON_BUILD_PATH: csp-lmc-common/docker
		#build:csp-lmc-common_image:
		# image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
		# before_script:
		# - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
		# tags:
		# - k8srunner
		# - engageska
		# stage: build_common
		# script:
		# - cd $COMMON_BUILD_PATH
		# - make build
		# variables:
		# COMMON_BUILD_PATH: csp-lmc-common/docker

		#
		# Test csp-lmc-common project
		#

		.test_common:
		test:csp-lmc-common:
		image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
		stage: test_common
		tags:
		- k8srunner
		artifacts:
		paths:
		- ./$COMMON_BUILD_PATH/build/
		variables:
		COMMON_BUILD_PATH: csp-lmc-common

		test:csp-lmc-common:
		extends: .test_common
		stage: test_common
		script:
		- echo $CI_REGISTRY_IMAGE
		- echo $CI_REGISTRY
		- echo $CI_REGISTRY_USER
		- cd $COMMON_BUILD_PATH
		- pwd
		- python3 -m pip install -r requirements-tst.txt
		- python3 -m pip install --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple -r requirements.txt .
		- pytest tests \|tee setup_py_test.stdout
		- python3 -m pip install -r requirements.txt .
		- python3 -m pytest tests \| tee setup_py_test.stdout
		- pwd
		- mkdir -p build/reports
		- ls -alR
		- mv setup_py_test.stdout build/csp-lmc-common-setup-test.stdout
		- mv htmlcov build/csp-lmc-common_htmlcov
		- mv coverage.xml build
		artifacts:
		paths:
		- ./$COMMON_BUILD_PATH/build/
		variables:
		COMMON_BUILD_PATH: csp-lmc-common
		#
		# linting stage

		@@ -117,58 +112,59 @@ linting:csp-lmc-common:
		.publish_python_common:
		stage: publish_common
		image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
		variables:
		TWINE_USERNAME: $TWINE_USERNAME
		TWINE_PASSWORD: $TWINE_PASSWORD
		tags:
		- k8srunner
		before_script:
		- python3 -m pip install twine
		variables:
		COMMON_BUILD_PATH: csp-lmc-common

		.publish_python_dev_common:
		extends: .publish_python_common
		except: [master]
		script:
		- cd $BUILD_PATH
		- twine upload --repository-url $PYPI_REPOSITORY_URL dist/* \|\| true

		.publish_python_release_common:
		extends: .publish_python_common
		only: [master]
		script:
		- cd $COMMON_BUILD_PATH
		- twine upload --repository-url $PYPI_REPOSITORY_URL dist/* \|\| true
		- twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* \|\| true

		# upload the common package to the Gitlab Package Repository.
		# Se the '--skip-existing' because if the CI/CD pipeline is re-run (without doing a commit)
		# the package version does not change and the upload to the repository fails.
		publish:csp-lmc-common_dev:
		extends: .publish_python_dev_common
		extends: .publish_python_common
		dependencies:
		- build:csp-lmc-common_pkg_dev
		variables:
		BUILD_PATH: csp-lmc-common
		script:
		- cd $COMMON_BUILD_PATH
		- TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --skip-existing --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $(ls -d ./dist/*.whl \| grep $CI_COMMIT_SHORT_SHA)

		# upload the package to the Nexus Repository only on project tagging.
		publish:csp-lmc-common:
		extends: .publish_python_release_common
		extends: .publish_python_common
		dependencies:
		- build:csp-lmc-common_pkg
		variables:
		COMMON_BUILD_PATH: csp-lmc-common
		TWINE_USERNAME: $TWINE_USERNAME
		TWINE_PASSWORD: $TWINE_PASSWORD
		only:
		refs:
		- tags
		script:
		- cd $COMMON_BUILD_PATH
		- twine upload --repository-url $PYPI_REPOSITORY_URL dist/* \|\| true
		- twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* \|\| true



		#
		# Release csp-lmc-common image on nexus
		#
		release csp-lmc-common image:
		tags:
		- k8srunner
		stage: release
		dependencies:
		- build:csp-lmc-common_image
		script:
		- cd $COMMON_BUILD_PATH
		- make push
		when: manual
		only:
		refs:
		- master
		variables:
		COMMON_BUILD_PATH: csp-lmc-common/docker
		#release csp-lmc-common image:
		# tags:
		# - k8srunner
		# stage: release
		# dependencies:
		# - build:csp-lmc-common_image
		# script:
		# - cd $COMMON_BUILD_PATH
		# - make push
		# when: manual
		# only:
		# refs:
		# - master
		# variables:
		# COMMON_BUILD_PATH: csp-lmc-common/docker

csp-lmc-common/HISTORY

+3 −0

Original line number	Diff line number	Diff line
		0.6.12
		- support to ADR18/22

		0.6.11
		- fix bug in setup.py

csp-lmc-common/csp_lmc_common/CspSubarray.py

+11 −14

Original line number	Diff line number	Diff line
		@@ -43,6 +43,7 @@ from ska.base.control_model import HealthState, AdminMode, ObsState, ObsMode
		from .utils.cspcommons import CmdExecState
		from .utils.decorators import transaction_id
		from . import release
		from .csp_manage_json import JsonConfiguration
		# PROTECTED REGION END # // CspSubarray.additionnal_import

		__all__ = ["CspSubarray", "main"]
		@@ -660,8 +661,6 @@ class CspSubarray(SKASubarray):
		self.logger.info("Configure ends with success!!")
		return target_device.configure_cmd_obj.succeeded()



		def validate_scan_configuration(self, argin):
		"""
		This method is overwritten in each CspSubarray instance.
		@@ -673,21 +672,19 @@ class CspSubarray(SKASubarray):

		target_device = self.target
		try:
		json_dict = json.loads(argin)
		if "id" not in json_dict:
		msg = ("Command failed: No configuration ID in JSON object")
		self.logger.error(msg)
		tango.Except.throw_exception("Command failed",
		msg,
		"ConfigureScan execution",
		tango.ErrSeverity.ERR)
		self._configuration_id = json_dict["id"]
		json_config = JsonConfiguration(argin, self.logger)
		target_device._configuration_id = json_config.get_id()
		self.logger.info("Processing configuration {}".format(target_device._configuration_id))

		except json.JSONDecodeError as e: # argument not a valid JSON object
		except Exception as e: # argument not a valid JSON object
		# this is a fatal error
		if isinstance(e, json.JSONDecodeError):
		msg = ("Scan configuration object is not a valid JSON object."
		"Aborting configuration:{}".format(str(e)))
		elif isinstance(e, ValueError):
		msg = "Command failed: No configuration ID in JSON object"
		else:
		msg = 'Unknown Error'
		self.logger.error(msg)
		tango.Except.throw_exception("Command failed",
		msg,

csp-lmc-common/csp_lmc_common/csp_manage_json.py

0 → 100644

+85 −0

Original line number	Diff line number	Diff line
		import json
		from ska_telmodel.csp.schema import validate_csp_config
		from ska_telmodel.csp.version import validate_csp_config_version

		class JsonConfiguration:
		"""
		A set of methods to manage the json configuration inside CSP code
		"""
		def __init__(self, json_config:str, logger):
		self.major_version = None
		self.minor_version = None
		self.logger = logger
		self.config = json.loads(json_config)

		def detect_version(self):
		"""
		Detect the version of the json configuration
		"""
		#ADR-22
		if 'interface' in self.config.keys():
		interface = self.config['interface']
		if validate_csp_config_version(interface):
		version_str = interface.split('/')[-1]
		self.major_version , self.minor_version = map(int, version_str.split('.'))
		else:
		message = 'Wrong json schema URI!'
		self.logger.error(message)
		raise ValueError(message)
		#ADR-18
		elif 'common' in self.config.keys():
		self.major_version, self.minor_version = (1,0)
		#pre ADR-18
		else:
		self.major_version, self.minor_version = (0,1)
		self.version = (self.major_version, self.minor_version)
		self.logger.info(f'Version is{self.major_version}{self.minor_version}')

		def get_section(self, name:str):
		"""
		Returns the correspondant section of json configuration.
		"""
		try:
		return self.config[name]
		except KeyError as e:
		self.logger.error(e)
		raise ValueError(e) from e

		def get_id(self):
		"""
		Return the json configuration id
		"""
		try:
		common_sec = self.get_section('common')
		config_id = common_sec['id']
		except ValueError as e:
		if 'id' in self.config:
		config_id = self.config['id']
		else:
		raise ValueError(e) from e
		return config_id

		def conversion_10_01(self):
		"""
		Converts the json from version 1.0 (ADR-18) to version 0.1.
		Note: Current Mid.CBF supports 0.1 Json schema.
		"""
		common_dict = self.get_section('common')
		cbf_dict = self.get_section('cbf')
		config_converted = {common_dict, cbf_dict}
		validate_csp_config(version=0, config=config_converted)
		self.config = config_converted

		def build_json_cbf(self) -> str:
		"""
		Returns the json configuration dictionary to be passed to cbf.
		If a version 0.2 is given, it converts to 0.1
		"""
		self.detect_version()
		if self.version == (0, 1):
		pass
		elif self.version == (1, 0):
		self.conversion_10_01()
		else:
		raise ValueError(f'version{self.version}is not supported')
		return json.dumps(self.config)