From b032f05ece975c6dd613fec51a420931f27a5b33 Mon Sep 17 00:00:00 2001
From: Gianluca Marotta <gianluca.marotta@inaf.it>
Date: Tue, 5 Jan 2021 19:45:39 +0100
Subject: [PATCH 01/46] CT-220 First implementation of manage json class

---
 .../csp_lmc_common/csp_manage_json.py         | 50 +++++++++++++++++++
 csp-lmc-common/requirements.txt               |  1 +
 2 files changed, 51 insertions(+)
 create mode 100644 csp-lmc-common/csp_lmc_common/csp_manage_json.py

diff --git a/csp-lmc-common/csp_lmc_common/csp_manage_json.py b/csp-lmc-common/csp_lmc_common/csp_manage_json.py
new file mode 100644
index 0000000..77e574d
--- /dev/null
+++ b/csp-lmc-common/csp_lmc_common/csp_manage_json.py
@@ -0,0 +1,50 @@
+import json
+from ska_telmodel.csp.schema import validate_csp_config
+
+class ManageJson:
+    def __init__(self, json_config:str, logger):
+        self.major_version=None
+        self.minor_version=None
+        self.config = json.loads(json_config)
+        self.logger = logger
+
+    def detect_version(self):
+        #ADR-22
+        if 'interface' in self.config.keys():
+            interface = self.config['interface']
+            version_str = interface.split('/')[-1]
+            self.major_version = int(version_str.split('.')[0])
+            self.minor_version = int(version_str.split('.')[1])
+        #ADR-18
+        elif 'common' in config.keys():
+            self.major_version = 0
+            self.minor_version = 2
+        #pre ADR-18
+        else: 
+            self.major_version = 0
+            self.minor_version = 1
+        self.version = [self.major_version, self.minor_version]
+        self.logger.info(f'Version is{major_version}{minor_version}')
+
+    def get_section(self, name:str):
+        try:
+            return self.config[name]
+        except KeyError as e:
+            self.logger.warning(e)
+            raise ValueError(e) from e
+
+    def conversion_02_01(self):
+        common_dict = self.get_section('common')
+        cbf_dict = self.get_section('cbf')
+        config_converted = {**common_dict, **cbf_dict}
+        validate_csp_config(0, config_converted)
+        self.config = config_converted
+            
+    def build_json(self):
+        if self.version == [0, 1]:
+            pass
+        elif self.version == [0,2]:
+            self.conversion_02_01()
+        else:
+            raise ValueError(f'version{self.version}is not supported')
+        return json.dumps(self.config)
diff --git a/csp-lmc-common/requirements.txt b/csp-lmc-common/requirements.txt
index e302923..b157404 100644
--- a/csp-lmc-common/requirements.txt
+++ b/csp-lmc-common/requirements.txt
@@ -3,3 +3,4 @@ pytango >= 9.3.2
 jsonschema >= 3.2.0
 lmcbaseclasses >= 0.7.2
 ska-log-transactions
+ska-telescope-model
-- 
GitLab


From 083636b5370cf0410344b4b399725a04db96fa39 Mon Sep 17 00:00:00 2001
From: Gianluca Marotta <gianluca.marotta@inaf.it>
Date: Wed, 6 Jan 2021 16:43:57 +0100
Subject: [PATCH 02/46] CT-220 Fixed Errors, added string documentation,
 changed class name

---
 .../csp_lmc_common/csp_manage_json.py         | 27 +++++++++++++++----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/csp-lmc-common/csp_lmc_common/csp_manage_json.py b/csp-lmc-common/csp_lmc_common/csp_manage_json.py
index 77e574d..82c1e5b 100644
--- a/csp-lmc-common/csp_lmc_common/csp_manage_json.py
+++ b/csp-lmc-common/csp_lmc_common/csp_manage_json.py
@@ -1,7 +1,10 @@
 import json
 from ska_telmodel.csp.schema import validate_csp_config
 
-class ManageJson:
+class JsonConfiguration:
+    """
+    A set of methods to manage the json configuration inside CSP code
+    """
     def __init__(self, json_config:str, logger):
         self.major_version=None
         self.minor_version=None
@@ -9,6 +12,9 @@ class ManageJson:
         self.logger = logger
 
     def detect_version(self):
+        """
+        Detect the version of the json configuration
+        """
         #ADR-22
         if 'interface' in self.config.keys():
             interface = self.config['interface']
@@ -16,7 +22,7 @@ class ManageJson:
             self.major_version = int(version_str.split('.')[0])
             self.minor_version = int(version_str.split('.')[1])
         #ADR-18
-        elif 'common' in config.keys():
+        elif 'common' in self.config.keys():
             self.major_version = 0
             self.minor_version = 2
         #pre ADR-18
@@ -24,9 +30,12 @@ class ManageJson:
             self.major_version = 0
             self.minor_version = 1
         self.version = [self.major_version, self.minor_version]
-        self.logger.info(f'Version is{major_version}{minor_version}')
+        self.logger.info(f'Version is{self.major_version}{self.minor_version}')
 
     def get_section(self, name:str):
+        """
+        Returns the correpondant section of json configuration
+        """
         try:
             return self.config[name]
         except KeyError as e:
@@ -34,13 +43,21 @@ class ManageJson:
             raise ValueError(e) from e
 
     def conversion_02_01(self):
+        """
+        Converts the json from version 0.2 to version 0.1
+        """
         common_dict = self.get_section('common')
         cbf_dict = self.get_section('cbf')
         config_converted = {**common_dict, **cbf_dict}
-        validate_csp_config(0, config_converted)
+        validate_csp_config(version=0, template=True, config=config_converted) #check template parameter
         self.config = config_converted
             
-    def build_json(self):
+    def build_json_cbf(self) -> dict:
+        "
+        Returns the json configuration dictionary to be passed to cbf.
+        If a version 0.2 is given, it converts to 0.1
+        "
+        self.detect_version()
         if self.version == [0, 1]:
             pass
         elif self.version == [0,2]:
-- 
GitLab


From 3798d0efc95efeaeeb5f614663d1385a56376738 Mon Sep 17 00:00:00 2001
From: Gianluca Marotta <gianluca.marotta@inaf.it>
Date: Fri, 8 Jan 2021 18:12:55 +0100
Subject: [PATCH 03/46] CT-220 Implemented unit test for jsonConfiguration
 class

---
 .../csp_lmc_common/csp_manage_json.py         |  48 ++---
 csp-lmc-common/tests/unit/json_config_test.py | 175 ++++++++++++++++++
 2 files changed, 201 insertions(+), 22 deletions(-)
 create mode 100644 csp-lmc-common/tests/unit/json_config_test.py

diff --git a/csp-lmc-common/csp_lmc_common/csp_manage_json.py b/csp-lmc-common/csp_lmc_common/csp_manage_json.py
index 82c1e5b..e61cd00 100644
--- a/csp-lmc-common/csp_lmc_common/csp_manage_json.py
+++ b/csp-lmc-common/csp_lmc_common/csp_manage_json.py
@@ -1,13 +1,14 @@
 import json
-from ska_telmodel.csp.schema import validate_csp_config
+from ska_telmodel.csp.schema import validate_csp_config 
+from ska_telmodel.csp.version import validate_csp_config_version
 
 class JsonConfiguration:
     """
     A set of methods to manage the json configuration inside CSP code
     """
     def __init__(self, json_config:str, logger):
-        self.major_version=None
-        self.minor_version=None
+        self.major_version = None
+        self.minor_version = None
         self.config = json.loads(json_config)
         self.logger = logger
 
@@ -18,50 +19,53 @@ class JsonConfiguration:
         #ADR-22
         if 'interface' in self.config.keys():
             interface = self.config['interface']
-            version_str = interface.split('/')[-1]
-            self.major_version = int(version_str.split('.')[0])
-            self.minor_version = int(version_str.split('.')[1])
+            if validate_csp_config_version(interface):
+                version_str = interface.split('/')[-1]
+                self.major_version , self.minor_version = map(int, version_str.split('.'))
+            else:
+                message = 'Wrong json schema URI!'
+                self.logger.error(message)
+                raise ValueError(message)
         #ADR-18
         elif 'common' in self.config.keys():
-            self.major_version = 0
-            self.minor_version = 2
+            self.major_version, self.minor_version = (1,0)
         #pre ADR-18
         else: 
-            self.major_version = 0
-            self.minor_version = 1
-        self.version = [self.major_version, self.minor_version]
+            self.major_version, self.minor_version = (0,1)
+        self.version = (self.major_version, self.minor_version)
         self.logger.info(f'Version is{self.major_version}{self.minor_version}')
 
     def get_section(self, name:str):
         """
-        Returns the correpondant section of json configuration
+        Returns the correspondant section of json configuration.
         """
         try:
             return self.config[name]
         except KeyError as e:
-            self.logger.warning(e)
+            self.logger.error(e)
             raise ValueError(e) from e
 
-    def conversion_02_01(self):
+    def conversion_10_01(self):
         """
-        Converts the json from version 0.2 to version 0.1
+        Converts the json from version 1.0 (ADR-18) to version 0.1.
+        Note: Current Mid.CBF supports 0.1 Json schema.
         """
         common_dict = self.get_section('common')
         cbf_dict = self.get_section('cbf')
         config_converted = {**common_dict, **cbf_dict}
-        validate_csp_config(version=0, template=True, config=config_converted) #check template parameter
+        validate_csp_config(version=0, config=config_converted) 
         self.config = config_converted
             
-    def build_json_cbf(self) -> dict:
-        "
+    def build_json_cbf(self) -> str:
+        """
         Returns the json configuration dictionary to be passed to cbf.
         If a version 0.2 is given, it converts to 0.1
-        "
+        """
         self.detect_version()
-        if self.version == [0, 1]:
+        if self.version == (0, 1):
             pass
-        elif self.version == [0,2]:
-            self.conversion_02_01()
+        elif self.version == (1, 0):
+            self.conversion_10_01()
         else:
             raise ValueError(f'version{self.version}is not supported')
         return json.dumps(self.config)
diff --git a/csp-lmc-common/tests/unit/json_config_test.py b/csp-lmc-common/tests/unit/json_config_test.py
new file mode 100644
index 0000000..52ab4de
--- /dev/null
+++ b/csp-lmc-common/tests/unit/json_config_test.py
@@ -0,0 +1,175 @@
+from csp_lmc_common.csp_manage_json import JsonConfiguration
+import logging
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+json_string_test_ADR_22 = """
+{
+    "interface": "https://schema.skatelescope.org/ska-csp-configure/1.0",
+    "common": {
+        "id": "sbi-mvp01-20200325-00001-science_A",     
+        "frequencyBand": "1"     
+    },
+    "cbf": {
+        "fsp": [       
+            {         
+                "fspID": 1, "functionMode": "CORR", "frequencySliceID": 1,         
+                "integrationTime": 1400, "corrBandwidth": 0,         
+                "channelAveragingMap": [[0,2], [744,0]],         
+                "fspChannelOffset": 0,         
+                "outputLinkMap": [[0,0], [200,1]]       
+            },       
+            {         
+                "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2,         
+                "integrationTime": 1400, "corrBandwidth": 0,         
+                "channelAveragingMap": [[0,2], [744,0]],         
+                "fspChannelOffset": 744,         
+                "outputLinkMap": [[0,4], [200,5]]       
+            }     
+        ] 
+    }
+}
+"""
+json_string_test_ADR_18 = """
+{
+    "common": {
+
+        "id": "sbi-mvp01-20200325-00001-science_A",     
+        "frequencyBand": "1"     
+    },
+    "cbf": {
+        "fsp": [       
+            {         
+                "fspID": 1, "functionMode": "CORR", "frequencySliceID": 1,         
+                "integrationTime": 1400, "corrBandwidth": 0,         
+                "channelAveragingMap": [[0,2], [744,0]],         
+                "fspChannelOffset": 0,         
+                "outputLinkMap": [[0,0], [200,1]]       
+            },       
+            {         
+                "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2,         
+                "integrationTime": 1400, "corrBandwidth": 0,         
+                "channelAveragingMap": [[0,2], [744,0]],         
+                "fspChannelOffset": 744,         
+                "outputLinkMap": [[0,4], [200,5]]       
+            }     
+        ] 
+    }
+}
+"""
+#function mode missing 
+json_string_test_ADR_18_invalid = """
+{
+    "common": {
+
+        "id": "sbi-mvp01-20200325-00001-science_A",     
+        "frequencyBand": "1", 
+    },
+    "cbf": {
+        "fsp": [       
+            {         
+                "fspID": 1, "frequencySliceID": 1,         
+                "integrationTime": 1400, "corrBandwidth": 0,         
+                "channelAveragingMap": [[0,2], [744,0]],         
+                "fspChannelOffset": 0,         
+                "outputLinkMap": [[0,0], [200,1]]       
+            },       
+            {         
+                "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2,         
+                "integrationTime": 1400, "corrBandwidth": 0,         
+                "channelAveragingMap": [[0,2], [744,0]],         
+                "fspChannelOffset": 744,         
+                "outputLinkMap": [[0,4], [200,5]]       
+            }     
+        ] 
+    }
+}
+"""
+json_string_test_pre_ADR_18 = """
+{
+    "id": "sbi-mvp01-20200325-00001-science_A",     
+    "frequencyBand": "1",     
+    "fsp": [       
+        {         
+            "fspID": 1, "functionMode": "CORR", "frequencySliceID": 1,         
+            "integrationTime": 1400, "corrBandwidth": 0,         
+            "channelAveragingMap": [[0,2], [744,0]],         
+            "fspChannelOffset": 0,         
+            "outputLinkMap": [[0,0], [200,1]]       
+        },       
+        {         
+            "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2,         
+            "integrationTime": 1400, "corrBandwidth": 0,         
+            "channelAveragingMap": [[0,2], [744,0]],         
+            "fspChannelOffset": 744,         
+            "outputLinkMap": [[0,4], [200,5]]       
+        }     
+    ] 
+}
+"""
+
+json_string_invalid_URI = '{"interface": "invaliduri.com/0.1"}'
+
+class TestJsonConfiguration:
+
+    def init_config(self):
+        self.json_config_ADR22 = JsonConfiguration(json_string_test_ADR_22, LOGGER)
+        self.json_config_ADR18 = JsonConfiguration(json_string_test_ADR_18, LOGGER)
+        self.json_config_pre_ADR18 = JsonConfiguration(json_string_test_pre_ADR_18, LOGGER)
+
+
+    def test_detect_version(self):
+        self.init_config()
+        self.json_config_ADR22.detect_version()
+        assert self.json_config_ADR22.major_version == 1  
+        assert self.json_config_ADR22.minor_version == 0
+
+        self.json_config_ADR18.detect_version()
+        assert self.json_config_ADR18.major_version == 1  
+        assert self.json_config_ADR18.minor_version == 0
+
+        self.json_config_pre_ADR18.detect_version()
+        assert self.json_config_pre_ADR18.major_version == 0  
+        assert self.json_config_pre_ADR18.minor_version == 1
+
+    def test_detect_version_with_invalid_uri(self):
+        with pytest.raises(ValueError, match= r'Wrong json schema URI!'):
+            JsonConfiguration(json_string_invalid_URI, LOGGER).detect_version()
+            
+    def test_get_section_when_section_is_present(self):
+        self.init_config()
+        assert isinstance(self.json_config_ADR18.get_section('cbf'), dict)
+        assert 'fsp' in self.json_config_ADR18.get_section('cbf').keys()
+
+        assert isinstance(self.json_config_pre_ADR18.get_section('fsp'), list)
+    
+    def test_get_section_when_section_is_not_present(self):
+        self.init_config()
+        with pytest.raises(ValueError):
+            self.json_config_pre_ADR18.get_section('cbf')
+        
+        with pytest.raises(ValueError):
+            self.json_config_ADR18.get_section('fsp')
+
+    def test_conversion_10_01(self):
+        self.init_config()
+        self.json_config_ADR18.conversion_10_01()
+        assert self.json_config_ADR18.config == self.json_config_pre_ADR18.config
+
+    def test_conversion_10_01_invalid_schema(self):
+        with pytest.raises(ValueError):
+            JsonConfiguration(json_string_test_ADR_18_invalid, LOGGER).conversion_10_01()
+
+    def test_build_json_cbf_version(self):
+        self.init_config()
+        assert self.json_config_ADR18.build_json_cbf() == self.json_config_pre_ADR18.build_json_cbf() 
+        assert self.json_config_ADR22.build_json_cbf() == self.json_config_pre_ADR18.build_json_cbf()
+
+    def test_build_json_cbf_version_not_supported(self):
+        self.init_config()
+        # version in interface is changed to 2.0 to force the failing of the test
+        self.json_config_ADR22.config['interface'] = "https://schema.skatelescope.org/ska-csp-configure/2.0"
+        with pytest.raises(ValueError, match=r'version.*is not supported'):
+            self.json_config_ADR22.build_json_cbf()
+
-- 
GitLab


From b96c5e43b9dc168fd1874d98fe47a10d1ce9c437 Mon Sep 17 00:00:00 2001
From: Gianluca Marotta <gianluca.marotta@inaf.it>
Date: Fri, 8 Jan 2021 19:12:08 +0100
Subject: [PATCH 04/46] modified test in pipeline to have coverage

---
 csp-lmc-common/.gitlab-ci.yml | 2 +-
 csp-lmc-common/setup.py       | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 3535b76..23fd087 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -78,7 +78,7 @@ test:csp-lmc-common:
     - pwd
     - python3 -m pip install -r requirements-tst.txt
     - python3 -m pip install --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple -r requirements.txt .
-    - pytest tests |tee setup_py_test.stdout
+    - python3 setup.py test 
     - pwd
     - mkdir -p build/reports
     - ls -alR
diff --git a/csp-lmc-common/setup.py b/csp-lmc-common/setup.py
index aa15c8f..62bb9f1 100644
--- a/csp-lmc-common/setup.py
+++ b/csp-lmc-common/setup.py
@@ -40,7 +40,6 @@ setup(
             'pytango >9.3.1',
             'future',
             'lmcbaseclasses',
-            'ska-log-transactions'
         ],
         dependency_links=[
             'https://nexus.engageska-portugal.pt/repository/pypi/simple'
@@ -48,7 +47,7 @@ setup(
         setup_requires=[
             'pytest-runner',
             'sphinx',
-            'recommonmark'
+            'recommonmark', 
         ],
         test_suite="test",
         entry_points={'console_scripts':[
-- 
GitLab


From 6300f9cfb7f06ba43943dc496ff30ad4320e7115 Mon Sep 17 00:00:00 2001
From: Gianluca Marotta <gianluca.marotta@inaf.it>
Date: Fri, 8 Jan 2021 19:22:19 +0100
Subject: [PATCH 05/46] fixed bug in pipeline

---
 csp-lmc-common/.gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 23fd087..f66bf59 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -78,7 +78,7 @@ test:csp-lmc-common:
     - pwd
     - python3 -m pip install -r requirements-tst.txt
     - python3 -m pip install --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple -r requirements.txt .
-    - python3 setup.py test 
+    - python3 setup.py test | tee setup_py_test.stdout  
     - pwd
     - mkdir -p build/reports
     - ls -alR
-- 
GitLab


From c45278bc1038f34b5b9badb5e01baeba820a40e4 Mon Sep 17 00:00:00 2001
From: GianlucaMarotta <gianluca.marotta@inaf.it>
Date: Mon, 11 Jan 2021 17:10:21 +0100
Subject: [PATCH 06/46] CT-220 Implemented get_id in JsonConfiguration class
 with test, Updated ConfigureScan method in CspSubarray.py

---
 csp-lmc-common/csp_lmc_common/CspSubarray.py  | 25 ++++++++-----------
 .../csp_lmc_common/csp_manage_json.py         | 16 +++++++++++-
 csp-lmc-common/tests/unit/json_config_test.py | 15 ++++++++++-
 3 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/csp-lmc-common/csp_lmc_common/CspSubarray.py b/csp-lmc-common/csp_lmc_common/CspSubarray.py
index 811da7e..3e0f3a8 100644
--- a/csp-lmc-common/csp_lmc_common/CspSubarray.py
+++ b/csp-lmc-common/csp_lmc_common/CspSubarray.py
@@ -43,6 +43,7 @@ from ska.base.control_model import HealthState, AdminMode, ObsState, ObsMode
 from .utils.cspcommons import CmdExecState
 from .utils.decorators import transaction_id
 from . import release
+from .csp_manage_json import JsonConfiguration
 # PROTECTED REGION END #    //  CspSubarray.additionnal_import
 
 __all__ = ["CspSubarray", "main"]
@@ -660,8 +661,6 @@ class CspSubarray(SKASubarray):
                     self.logger.info("Configure ends with success!!")
                     return target_device.configure_cmd_obj.succeeded()
 
-                
-
         def validate_scan_configuration(self, argin):
             """
             This method is overwritten in each CspSubarray instance.
@@ -673,21 +672,19 @@ class CspSubarray(SKASubarray):
 
             target_device = self.target
             try:
-                json_dict = json.loads(argin)
-                if "id" not in json_dict:
-                    msg = ("Command failed: No configuration ID in JSON object")
-                    self.logger.error(msg)
-                    tango.Except.throw_exception("Command failed",
-                                                 msg,
-                                                 "ConfigureScan execution",
-                                                 tango.ErrSeverity.ERR)
-                self._configuration_id = json_dict["id"]
+                json_config = JsonConfiguration(argin, self.logger)
+                target_device._configuration_id = json_config.get_id()
                 self.logger.info("Processing configuration {}".format(target_device._configuration_id))
 
-            except json.JSONDecodeError as e:  # argument not a valid JSON object
+            except Exception as e:  # argument not a valid JSON object
                 # this is a fatal error
-                msg = ("Scan configuration object is not a valid JSON object."
-                       "Aborting configuration:{}".format(str(e)))
+                if isinstance(e, json.JSONDecodeError):
+                    msg = ("Scan configuration object is not a valid JSON object."
+                           "Aborting configuration:{}".format(str(e)))
+                elif isinstance(e, ValueError):
+                    msg = "Command failed: No configuration ID in JSON object"
+                else:
+                    msg = 'Unknown Error'
                 self.logger.error(msg)
                 tango.Except.throw_exception("Command failed",
                                              msg,
diff --git a/csp-lmc-common/csp_lmc_common/csp_manage_json.py b/csp-lmc-common/csp_lmc_common/csp_manage_json.py
index e61cd00..9d88f01 100644
--- a/csp-lmc-common/csp_lmc_common/csp_manage_json.py
+++ b/csp-lmc-common/csp_lmc_common/csp_manage_json.py
@@ -9,8 +9,8 @@ class JsonConfiguration:
     def __init__(self, json_config:str, logger):
         self.major_version = None
         self.minor_version = None
-        self.config = json.loads(json_config)
         self.logger = logger
+        self.config = json.loads(json_config)
 
     def detect_version(self):
         """
@@ -45,6 +45,20 @@ class JsonConfiguration:
             self.logger.error(e)
             raise ValueError(e) from e
 
+    def get_id(self):
+        """
+        Return the json configuration id
+        """
+        try:
+            common_sec = self.get_section('common')
+            config_id = common_sec['id']
+        except ValueError as e:
+            if 'id' in self.config:
+                config_id = self.config['id']
+            else:
+                raise ValueError(e) from e
+        return config_id
+
     def conversion_10_01(self):
         """
         Converts the json from version 1.0 (ADR-18) to version 0.1.
diff --git a/csp-lmc-common/tests/unit/json_config_test.py b/csp-lmc-common/tests/unit/json_config_test.py
index 52ab4de..b2dcf50 100644
--- a/csp-lmc-common/tests/unit/json_config_test.py
+++ b/csp-lmc-common/tests/unit/json_config_test.py
@@ -1,3 +1,4 @@
+import json
 from csp_lmc_common.csp_manage_json import JsonConfiguration
 import logging
 import pytest
@@ -64,7 +65,7 @@ json_string_test_ADR_18_invalid = """
     "common": {
 
         "id": "sbi-mvp01-20200325-00001-science_A",     
-        "frequencyBand": "1", 
+        "frequencyBand": "1" 
     },
     "cbf": {
         "fsp": [       
@@ -173,3 +174,15 @@ class TestJsonConfiguration:
         with pytest.raises(ValueError, match=r'version.*is not supported'):
             self.json_config_ADR22.build_json_cbf()
 
+    def test_get_id(self):
+        self.init_config()
+        assert self.json_config_pre_ADR18.get_id() == "sbi-mvp01-20200325-00001-science_A"
+        assert self.json_config_ADR18.get_id() == "sbi-mvp01-20200325-00001-science_A"
+
+    def test_get_id_NOT_PRESENT(self):
+        self.init_config()
+        self.json_config_pre_ADR18.config.pop('id')
+        json_config_with_no_id = json.dumps(self.json_config_pre_ADR18.config)
+        json_config_with_no_id = JsonConfiguration(json_config_with_no_id, LOGGER)
+        with pytest.raises(ValueError):
+            json_config_with_no_id.get_id()
-- 
GitLab


From ed3d8a5df610c2428a0c6fd9072eafcc7b55884f Mon Sep 17 00:00:00 2001
From: GianlucaMarotta <gianluca.marotta@inaf.it>
Date: Mon, 11 Jan 2021 18:46:30 +0100
Subject: [PATCH 07/46] CT-220 Implemented support ADR18/22 to midCSP-LMC;
 update releases

---
 csp-lmc-common/HISTORY                        |  3 ++
 csp-lmc-common/csp_lmc_common/release.py      |  2 +-
 csp-lmc-common/setup.py                       |  1 +
 csp-lmc-mid/.make/release.mk                  |  2 +-
 csp-lmc-mid/.release                          |  6 ++--
 csp-lmc-mid/HISTORY                           |  4 +++
 csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py |  4 ++-
 csp-lmc-mid/csp_lmc_mid/release.py            |  2 +-
 csp-lmc-mid/requirements.txt                  |  7 +++--
 csp-lmc-mid/setup.py                          |  3 +-
 .../test_data/test_ConfigureScan_ADR22.json   | 28 +++++++++++++++++++
 .../tests/unit/midcspsubarray_unit_test.py    |  2 +-
 12 files changed, 52 insertions(+), 12 deletions(-)
 create mode 100644 csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json

diff --git a/csp-lmc-common/HISTORY b/csp-lmc-common/HISTORY
index 1a8cca1..b7bd5ba 100644
--- a/csp-lmc-common/HISTORY
+++ b/csp-lmc-common/HISTORY
@@ -1,3 +1,6 @@
+0.6.12
+ - support to ADR18/22
+
 0.6.11
  - fix bug in setup.py
 
diff --git a/csp-lmc-common/csp_lmc_common/release.py b/csp-lmc-common/csp_lmc_common/release.py
index 66666d6..28b5231 100755
--- a/csp-lmc-common/csp_lmc_common/release.py
+++ b/csp-lmc-common/csp_lmc_common/release.py
@@ -10,7 +10,7 @@
 """Release information for Python Package"""
 
 name = """csp-lmc-common"""
-version = "0.6.11"
+version = "0.6.12"
 version_info = version.split(".")
 description = """SKA CSP.LMC Common Software"""
 author = "INAF-OAA"
diff --git a/csp-lmc-common/setup.py b/csp-lmc-common/setup.py
index 62bb9f1..b9315af 100644
--- a/csp-lmc-common/setup.py
+++ b/csp-lmc-common/setup.py
@@ -40,6 +40,7 @@ setup(
             'pytango >9.3.1',
             'future',
             'lmcbaseclasses',
+            'ska-telescope-model'
         ],
         dependency_links=[
             'https://nexus.engageska-portugal.pt/repository/pypi/simple'
diff --git a/csp-lmc-mid/.make/release.mk b/csp-lmc-mid/.make/release.mk
index e1597a4..2ac90c4 100644
--- a/csp-lmc-mid/.make/release.mk
+++ b/csp-lmc-mid/.make/release.mk
@@ -42,7 +42,7 @@ BASE_RELEASE=$(shell . $(RELEASE_SUPPORT) ;getRelease)
 BASE_VERSION=$(shell . $(RELEASE_SUPPORT) ;getBaseVersion)
 
 ifeq ($(BRANCH_NAME), master)
-IMAGE_TAG ?= $(BASE_RELEASE)
+IMAGE_TAG ?= $(BASE_VERSION)
 else
 IMAGE_TAG ?= $(BASE_VERSION)-dev
 endif
diff --git a/csp-lmc-mid/.release b/csp-lmc-mid/.release
index f5a4952..8ad0f4b 100644
--- a/csp-lmc-mid/.release
+++ b/csp-lmc-mid/.release
@@ -9,7 +9,7 @@
 """Release information for Python Package"""
 
 name = """MID CSP.LMC"""
-version = "0.6.9"
+version = "0.6.10"
 version_info = version.split(".")
 description = """SKA MID CSP.LMC Classes"""
 author = "E.G"
@@ -18,5 +18,5 @@ license = """BSD-3-Clause"""
 url = """www.tango-controls.org"""
 copyright = """"""
 
-release=0.6.9
-tag=mid-csp-lmc-0.6.9
+release=0.6.10
+tag=mid-csp-lmc-0.6.10
diff --git a/csp-lmc-mid/HISTORY b/csp-lmc-mid/HISTORY
index 4de4f75..428141f 100644
--- a/csp-lmc-mid/HISTORY
+++ b/csp-lmc-mid/HISTORY
@@ -1,3 +1,7 @@
+0.6.10
+- use csp-lmc-common 0.6.12
+- support to ADR-18/22
+
 0.6.9:
 - use csp-lmc-common 0.6.9
 - fix bug in receptor class
diff --git a/csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py b/csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py
index 21e65a4..d517ba4 100644
--- a/csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py
+++ b/csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py
@@ -47,6 +47,7 @@ from csp_lmc_common.utils.cspcommons import CmdExecState
 #from csp_lmc_common.CspSubarray import CspSubarray, CspSubarrayStateModel
 from csp_lmc_common.CspSubarray import CspSubarray
 from csp_lmc_mid.receptors import Receptors
+from csp_lmc_common.csp_manage_json import JsonConfiguration
 # PROTECTED REGION END #    //  MidCspSubarrayBase.additionnal_import
 
 __all__ = ["MidCspSubarrayBase", "main"]
@@ -392,7 +393,8 @@ class MidCspSubarrayBase(CspSubarray):
             self.logger.debug("Validate scan configuration for MID CSP")
             #self._sc_subarray_obs_mode[self.CbfSubarray] = ObsMode.IMAGING
             #self._sc_scan_configuration[self.CbfSubarray] = json_config["cbf"]
-            device._sc_subarray_scan_configuration[device.CbfSubarray] = argin
+            json_config = JsonConfiguration(argin, self.logger)
+            device._sc_subarray_scan_configuration[device.CbfSubarray] = json_config.build_json_cbf()
             self.logger.debug("Validate scan: {}".format(device._sc_subarray_assigned_fqdn))
 
     def __len__(self):
diff --git a/csp-lmc-mid/csp_lmc_mid/release.py b/csp-lmc-mid/csp_lmc_mid/release.py
index 102b81e..1302a24 100755
--- a/csp-lmc-mid/csp_lmc_mid/release.py
+++ b/csp-lmc-mid/csp_lmc_mid/release.py
@@ -10,7 +10,7 @@
 """Release information for Python Package"""
 
 name = """mid-csp-lmc"""
-version = "0.6.9"
+version = "0.6.10"
 version_info = version.split(".")
 description = """SKA MID CSP.LMC"""
 author = "INAF-OAA"
diff --git a/csp-lmc-mid/requirements.txt b/csp-lmc-mid/requirements.txt
index 5a67db3..9f4258b 100644
--- a/csp-lmc-mid/requirements.txt
+++ b/csp-lmc-mid/requirements.txt
@@ -1,5 +1,8 @@
 numpy == 1.17.2
 pytango >= 9.3.2
 jsonschema >= 3.2.0
-lmcbaseclasses >= 0.6.4
-csp-lmc-common >= 0.6.8
+lmcbaseclasses >= 0.7.2
+csp-lmc-common >= 0.6.12
+ska-log-transactions
+ska-telescope-model
+
diff --git a/csp-lmc-mid/setup.py b/csp-lmc-mid/setup.py
index 14177cd..09ee7ed 100644
--- a/csp-lmc-mid/setup.py
+++ b/csp-lmc-mid/setup.py
@@ -37,8 +37,7 @@ setup(
         include_package_data=True,
         install_requires = [
             'pytango >=9.3.1',
-            'future',
-            'csp-lmc-common>=0.6.9'
+            'future'
         ],
         setup_requires=[
             'pytest-runner',
diff --git a/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json b/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json
new file mode 100644
index 0000000..7f3b0ff
--- /dev/null
+++ b/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json
@@ -0,0 +1,28 @@
+{
+    "interface": "https://schema.skatelescope.org/ska-csp-configure/1.0",
+    "common": {
+        "id": "sbi-mvp01-20200325-00001-science_A",
+        "frequencyBand": "1"
+    },
+    "cbf": {
+        "fsp": [
+            {
+                "fspID": 1, "functionMode": "CORR", "frequencySliceID": 1,
+                "integrationTime": 1400, "corrBandwidth": 0,
+                "channelAveragingMap": [[0,2], [744,0]],
+                "fspChannelOffset": 0,
+                "outputLinkMap": [[0,0], [200,1]],
+                "outputHost": [[0, "192.168.0.1"], [8184, "192.168.0.2"]],
+                "outputMac": [[0, "06-00-00-00-00-01"]],
+                "outputPort": [[0, 9000, 1], [8184, 9000, 1]]
+            },
+            {
+                "fspID": 2, "functionMode": "CORR", "frequencySliceID": 2,
+                "integrationTime": 1400, "corrBandwidth": 0,
+                "channelAveragingMap": [[0,2], [744,0]],
+                "fspChannelOffset": 744,
+                "outputLinkMap": [[0,4], [200,5]]
+            }
+        ]
+    }
+}
\ No newline at end of file
diff --git a/csp-lmc-mid/tests/unit/midcspsubarray_unit_test.py b/csp-lmc-mid/tests/unit/midcspsubarray_unit_test.py
index 5867030..1f0b9ef 100644
--- a/csp-lmc-mid/tests/unit/midcspsubarray_unit_test.py
+++ b/csp-lmc-mid/tests/unit/midcspsubarray_unit_test.py
@@ -372,7 +372,7 @@ def test_midcspsubarray_obsstate_AFTER_configure():
             tango_context.device.AddReceptors(receptor_list)
             #assert tango_context.device.obsState == ObsState.IDLE
             cbf_subarray_device_proxy_mock.configureDelay.return_value = 10
-            configuration_string = load_json_file("test_ConfigureScan_ADR4.json")
+            configuration_string = load_json_file("test_ConfigureScan_ADR22.json")
             tango_context.device.Configure(configuration_string)
             #assert tango_context.device.obsState == ObsState.CONFIGURING
             dummy_event = create_dummy_event(cbf_subarray_fqdn, 'obsstate',ObsState.READY)
-- 
GitLab


From d74a8ae5fa4366b20fe3a80aa2acf617d1fbe676 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Mon, 11 Jan 2021 19:48:35 +0100
Subject: [PATCH 08/46] CT-220: Fix error in Dockerfile. Install python
 packages from requirements.txt file

---
 csp-lmc-mid/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/csp-lmc-mid/Dockerfile b/csp-lmc-mid/Dockerfile
index 2456a5e..d2b1bea 100644
--- a/csp-lmc-mid/Dockerfile
+++ b/csp-lmc-mid/Dockerfile
@@ -5,6 +5,6 @@ FROM nexus.engageska-portugal.pt/ska-docker/ska-python-runtime:9.3.2 AS runtime
 RUN ipython profile create
 
 ENV PATH=/home/tango/.local/bin:$PATH
-RUN python3 -m pip install . --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple
+RUN python3 -m pip install -r requirements.txt .
 
 CMD ["/venv/bin/python","/app/csp_lmc_mid/MidCspMaster.py"]
-- 
GitLab


From f46de3dd3383ee754abf06ac09b3ab960efe9a98 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Mon, 11 Jan 2021 20:47:09 +0100
Subject: [PATCH 09/46] CT-220: Add test for ADR22 support. Modified json
 configuration file adding the 'delayModelSubscriptionPoint' entry (is not
 optional but needed by mid-cbf)

---
 .../tests/integration/MidCspSubarray_test.py  | 23 ++++++++++++++++++-
 .../test_data/test_ConfigureScan_ADR22.json   |  7 +++---
 2 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/csp-lmc-mid/tests/integration/MidCspSubarray_test.py b/csp-lmc-mid/tests/integration/MidCspSubarray_test.py
index df27243..8bd8660 100755
--- a/csp-lmc-mid/tests/integration/MidCspSubarray_test.py
+++ b/csp-lmc-mid/tests/integration/MidCspSubarray_test.py
@@ -35,7 +35,7 @@ def prepare_configuration_string(filename="test_ConfigureScan_ADR4.json"):
     """Create the config string for CSP-CBF"""
     try:
         file_to_load = file_path + '/../test_data/' + filename
-        LOGGER.debug(f" Reading configuration from {file_to_load}")
+        LOGGER.info(f" Reading configuration from {file_to_load}")
         json_file = open(file_to_load)
         configuration_string = json_file.read().replace("\n", "")
         return configuration_string
@@ -406,6 +406,27 @@ class TestCspSubarray(TestBase):
         #stored_id = self.midcsp_subarray01.configurationID
         #assert stored_id == configID
 
+    @pytest.mark.csp_k8s
+    def test_send_configure_WITH_ADR22_json(self):
+        """
+        Configure the CSP Subarray with a JSon string including
+        the new ADR4 fields.
+        """
+        self._setup_subarray()
+        self._assign_receptors()
+        state = self.midcsp_subarray01.State()
+        obs_state = self.midcsp_subarray01.obsState
+        LOGGER.info("CSP Subarray State before exercise :{}-{}".format(state, ObsState(obs_state).name))
+        # exercise the device
+        LOGGER.info(f"Configuring CSP subarray01")
+        configuration_string = prepare_configuration_string("test_ConfigureScan_ADR22.json")
+        (result_code, msg) = self.midcsp_subarray01.Configure(configuration_string)
+        # check
+        prober_subarray_obstate = Probe(self.midcsp_subarray01, 'obsState', ObsState.READY,
+                                        f"Wrong CSP Subarray obsState {self.midcsp_subarray01.obsState}")
+        Poller(5, 0.2).check(prober_subarray_obstate)
+        obs_state = self.midcsp_subarray01.obsState
+
     @pytest.mark.csp_k8s
     def test_start_end_scan(self):
         """
diff --git a/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json b/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json
index 7f3b0ff..a326dd8 100644
--- a/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json
+++ b/csp-lmc-mid/tests/test_data/test_ConfigureScan_ADR22.json
@@ -1,10 +1,11 @@
 {
     "interface": "https://schema.skatelescope.org/ska-csp-configure/1.0",
     "common": {
-        "id": "sbi-mvp01-20200325-00001-science_A",
-        "frequencyBand": "1"
+        "id": "sbi-mvp01-20200326-00001-science_B",
+        "frequencyBand": "3"
     },
     "cbf": {
+    	"delayModelSubscriptionPoint": "ska_mid/tm_leaf_node/csp_subarray_01/delayModel",
         "fsp": [
             {
                 "fspID": 1, "functionMode": "CORR", "frequencySliceID": 1,
@@ -25,4 +26,4 @@
             }
         ]
     }
-}
\ No newline at end of file
+}
-- 
GitLab


From 6e07b1959d5090380b79f04514920a78e92989b5 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 07:33:13 +0100
Subject: [PATCH 10/46] TestPipeline: try to user docker registry to store
 image - first step

---
 csp-lmc-common/.gitlab-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index f66bf59..97546f7 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -74,6 +74,9 @@ test:csp-lmc-common:
   extends: .test_common
   stage: test_common
   script:
+    - echo $CI_REGISTRY_IMAGE
+    - echo $CI_REGISTRY
+    - echo $CI_REGISTRY_USER
     - cd $COMMON_BUILD_PATH
     - pwd
     - python3 -m pip install -r requirements-tst.txt
-- 
GitLab


From d5fcd3f34506ed14b2d4bbb14163019a68247d67 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 08:31:03 +0100
Subject: [PATCH 11/46] TestPipeline: Try to publish the python test package to
 gitlab

---
 csp-lmc-common/.gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 97546f7..34fbdbb 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -133,7 +133,8 @@ linting:csp-lmc-common:
   except: [master]
   script:
     - cd $BUILD_PATH
-    - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true
+    - cat $TESTPIPRC  > /tmp/.pypirc
+    - twine upload --repository-url gitlab $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA) --config-file /tmp/.pypirc
 
 .publish_python_release_common:
   extends: .publish_python_common
-- 
GitLab


From 965d661866041b4deb014b4f71f6f61c1012b5e4 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 09:13:45 +0100
Subject: [PATCH 12/46] TestPipeline: fix error in script to updaload package

---
 csp-lmc-common/.gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 34fbdbb..11a9100 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -134,7 +134,7 @@ linting:csp-lmc-common:
   script:
     - cd $BUILD_PATH
     - cat $TESTPIPRC  > /tmp/.pypirc
-    - twine upload --repository-url gitlab $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA) --config-file /tmp/.pypirc
+    - twine upload --repository gitlab --config-file /tmp/.pypirc $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
 
 .publish_python_release_common:
   extends: .publish_python_common
-- 
GitLab


From 51bed272e4eb08acbec496f30a8c7a5dca700c62 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 11:41:33 +0100
Subject: [PATCH 13/46] TestPipeline: Print some variables and use --verbose
 option for twine

---
 csp-lmc-common/.gitlab-ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 11a9100..908ec51 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -134,7 +134,9 @@ linting:csp-lmc-common:
   script:
     - cd $BUILD_PATH
     - cat $TESTPIPRC  > /tmp/.pypirc
-    - twine upload --repository gitlab --config-file /tmp/.pypirc $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
+    - echo $CSP_TOKEN
+    - echo $CI_API_V4_URL
+    - twine upload --repository gitlab --config-file /tmp/.pypirc --verbose$(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
 
 .publish_python_release_common:
   extends: .publish_python_common
-- 
GitLab


From 5c2ba67286a5de2369e82ede360d279ad3bbaa9a Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 11:47:46 +0100
Subject: [PATCH 14/46] TestPipeline: fix bug in twine command.

---
 csp-lmc-common/.gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 908ec51..5973ae0 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -136,7 +136,7 @@ linting:csp-lmc-common:
     - cat $TESTPIPRC  > /tmp/.pypirc
     - echo $CSP_TOKEN
     - echo $CI_API_V4_URL
-    - twine upload --repository gitlab --config-file /tmp/.pypirc --verbose$(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
+    - twine upload --repository gitlab --config-file /tmp/.pypirc --verbose $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
 
 .publish_python_release_common:
   extends: .publish_python_common
-- 
GitLab


From f27924ac63ce7073c98d85862aa6b828ae463ead Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 13:02:09 +0100
Subject: [PATCH 15/46] TestPipeline: Modified the script to upload the python
 package.

---
 csp-lmc-common/.gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 5973ae0..ce4e66a 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -136,7 +136,8 @@ linting:csp-lmc-common:
     - cat $TESTPIPRC  > /tmp/.pypirc
     - echo $CSP_TOKEN
     - echo $CI_API_V4_URL
-    - twine upload --repository gitlab --config-file /tmp/.pypirc --verbose $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
+    - python setup.py sdist bdist_wheel    
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/*
 
 .publish_python_release_common:
   extends: .publish_python_common
-- 
GitLab


From d757a1592dd279d70cd60eeda75dc2b6a0e06dc5 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 13:08:47 +0100
Subject: [PATCH 16/46] TestPipeline: fix error (need to call python3)

---
 csp-lmc-common/.gitlab-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index ce4e66a..9bd9138 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -136,8 +136,8 @@ linting:csp-lmc-common:
     - cat $TESTPIPRC  > /tmp/.pypirc
     - echo $CSP_TOKEN
     - echo $CI_API_V4_URL
-    - python setup.py sdist bdist_wheel    
-    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/*
+    - python3 setup.py sdist bdist_wheel    
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/*
 
 .publish_python_release_common:
   extends: .publish_python_common
-- 
GitLab


From 5883cb9fcf1c728049c788b8d15a7167a1aa7d61 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 13:27:11 +0100
Subject: [PATCH 17/46] TestPipeline: some changes to set variables.

---
 csp-lmc-common/.gitlab-ci.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 9bd9138..5c3bce7 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -130,14 +130,16 @@ linting:csp-lmc-common:
 
 .publish_python_dev_common:
   extends: .publish_python_common
+  variables:
+    TWINE_USERNAME: gitlab-ci-token
+    TWINE_PASSWORD: $CI_JOB_TOKEN
+    PACKAGE: $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
   except: [master]
   script:
     - cd $BUILD_PATH
     - cat $TESTPIPRC  > /tmp/.pypirc
-    - echo $CSP_TOKEN
-    - echo $CI_API_V4_URL
-    - python3 setup.py sdist bdist_wheel    
-    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi dist/*
+    - echo $PACKAGE
+    - twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi  --verbose $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
 
 .publish_python_release_common:
   extends: .publish_python_common
-- 
GitLab


From e12c7e6276c2d36628a02d1aed67cc212092b3a8 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 14:39:41 +0100
Subject: [PATCH 18/46] TestPipeline: removed TWINE variables from the stage.

---
 csp-lmc-common/.gitlab-ci.yml | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 5c3bce7..bc317cc 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -120,9 +120,6 @@ linting:csp-lmc-common:
 .publish_python_common:
   stage: publish_common
   image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
-  variables:
-    TWINE_USERNAME: $TWINE_USERNAME
-    TWINE_PASSWORD: $TWINE_PASSWORD
   tags:
     - k8srunner
   before_script:
@@ -130,19 +127,17 @@ linting:csp-lmc-common:
 
 .publish_python_dev_common:
   extends: .publish_python_common
-  variables:
-    TWINE_USERNAME: gitlab-ci-token
-    TWINE_PASSWORD: $CI_JOB_TOKEN
-    PACKAGE: $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
   except: [master]
   script:
     - cd $BUILD_PATH
     - cat $TESTPIPRC  > /tmp/.pypirc
-    - echo $PACKAGE
-    - twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi  --verbose $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
+    - twine upload --repository gitlab --config-file /tmp/.pypirc --verbose $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
 
 .publish_python_release_common:
   extends: .publish_python_common
+  variables:
+    TWINE_USERNAME: $TWINE_USERNAME
+    TWINE_PASSWORD: $TWINE_PASSWORD
   only: [master]
   script:
     - cd $COMMON_BUILD_PATH
-- 
GitLab


From 9e1d6450e2e17c1a0d0c52bd841c83bae7650658 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 15:03:23 +0100
Subject: [PATCH 19/46] TestPipeline: another test...

---
 csp-lmc-common/.gitlab-ci.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index bc317cc..30ae5a8 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -120,6 +120,9 @@ linting:csp-lmc-common:
 .publish_python_common:
   stage: publish_common
   image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
+  variables:
+    TWINE_USERNAME: $TWINE_USERNAME
+    TWINE_PASSWORD: $TWINE_PASSWORD
   tags:
     - k8srunner
   before_script:
@@ -128,16 +131,16 @@ linting:csp-lmc-common:
 .publish_python_dev_common:
   extends: .publish_python_common
   except: [master]
+  variables:
+    PACKAGE: $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
   script:
+    - PACKAGE = $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
     - cd $BUILD_PATH
-    - cat $TESTPIPRC  > /tmp/.pypirc
-    - twine upload --repository gitlab --config-file /tmp/.pypirc --verbose $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
+    - echo $PACKAGE
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $PACKAGE 
 
 .publish_python_release_common:
   extends: .publish_python_common
-  variables:
-    TWINE_USERNAME: $TWINE_USERNAME
-    TWINE_PASSWORD: $TWINE_PASSWORD
   only: [master]
   script:
     - cd $COMMON_BUILD_PATH
-- 
GitLab


From fac6c5116a488601ac7bc38d775c2435dab92e25 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 15:14:05 +0100
Subject: [PATCH 20/46] TestPipeline:removed variable

---
 csp-lmc-common/.gitlab-ci.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 30ae5a8..a061866 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -132,13 +132,11 @@ linting:csp-lmc-common:
   extends: .publish_python_common
   except: [master]
   variables:
-    PACKAGE: $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
   script:
     - PACKAGE = $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
     - cd $BUILD_PATH
     - echo $PACKAGE
-    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $PACKAGE 
-
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
 .publish_python_release_common:
   extends: .publish_python_common
   only: [master]
-- 
GitLab


From 3a3f4a1a1f8c3968585167c3ac6f35cd53a77314 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 15:19:18 +0100
Subject: [PATCH 21/46] TestPipeline: removed variable

---
 csp-lmc-common/.gitlab-ci.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index a061866..68b6154 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -131,11 +131,8 @@ linting:csp-lmc-common:
 .publish_python_dev_common:
   extends: .publish_python_common
   except: [master]
-  variables:
   script:
-    - PACKAGE = $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
     - cd $BUILD_PATH
-    - echo $PACKAGE
     - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
 .publish_python_release_common:
   extends: .publish_python_common
-- 
GitLab


From 3e723b2681acb651c518728217031da2dc59be89 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 16:57:36 +0100
Subject: [PATCH 22/46] TestPipeline: removed ska-telescope-model from setup.py

---
 csp-lmc-common/setup.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/csp-lmc-common/setup.py b/csp-lmc-common/setup.py
index b9315af..62bb9f1 100644
--- a/csp-lmc-common/setup.py
+++ b/csp-lmc-common/setup.py
@@ -40,7 +40,6 @@ setup(
             'pytango >9.3.1',
             'future',
             'lmcbaseclasses',
-            'ska-telescope-model'
         ],
         dependency_links=[
             'https://nexus.engageska-portugal.pt/repository/pypi/simple'
-- 
GitLab


From 23c8c19878b1cc9c864740415b52f2212e9320d9 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 17:33:03 +0100
Subject: [PATCH 23/46] TestPipeline: added all package dependencies to
 setup.py. Use 'python3 -m pytest- to run tests to get coverage.

---
 csp-lmc-common/.gitlab-ci.yml | 4 ++--
 csp-lmc-common/setup.py       | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 68b6154..469c1a1 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -80,8 +80,8 @@ test:csp-lmc-common:
     - cd $COMMON_BUILD_PATH
     - pwd
     - python3 -m pip install -r requirements-tst.txt
-    - python3 -m pip install --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple -r requirements.txt .
-    - python3 setup.py test | tee setup_py_test.stdout  
+    - python3 -m pip install -r requirements.txt .
+    - python3 -m pytest tests | tee setup_py_test.stdout  
     - pwd
     - mkdir -p build/reports
     - ls -alR
diff --git a/csp-lmc-common/setup.py b/csp-lmc-common/setup.py
index 62bb9f1..b7ff480 100644
--- a/csp-lmc-common/setup.py
+++ b/csp-lmc-common/setup.py
@@ -40,6 +40,8 @@ setup(
             'pytango >9.3.1',
             'future',
             'lmcbaseclasses',
+            'ska-telescope-model',
+            'ska-log-transactions'
         ],
         dependency_links=[
             'https://nexus.engageska-portugal.pt/repository/pypi/simple'
-- 
GitLab


From d336a4a49b416ea528157bb5d1f2766762479b5b Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 17:52:17 +0100
Subject: [PATCH 24/46] TestPipeline: Try to generate midcsp development image
 using the csp-lmc-common package in gitlab and to store the built image in
 gitlab repo for test purposes.

---
 csp-lmc-mid/.gitlab-ci.yml          | 36 ++++++++++++++++-------------
 csp-lmc-mid/Dockerfile.gitlab       | 16 +++++++++++++
 csp-lmc-mid/requirements-gitlab.txt |  7 ++++++
 3 files changed, 43 insertions(+), 16 deletions(-)
 create mode 100644 csp-lmc-mid/Dockerfile.gitlab
 create mode 100644 csp-lmc-mid/requirements-gitlab.txt

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 5677fe8..6916eb4 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -14,26 +14,15 @@
     paths:
       - ./$BUILD_PATH/dist/
 
-.build_python_dev:
-  extends: .build_python
-  dependencies: []
-  script:
-    - cd $BUILD_PATH
-    - python3 setup.py egg_info -b+dev.$CI_COMMIT_SHORT_SHA sdist bdist_wheel
-
-.build_python_release:
-  extends: .build_python
-  script:
-    - cd $BUILD_PATH
-    - python3 setup.py egg_info -b+$CI_COMMIT_SHORT_SHA sdist bdist_wheel
-
 build:csp-lmc-mid_pkg_dev: # Executed on a commit
-  extends: .build_python_dev
   variables:
     BUILD_PATH: csp-lmc-mid
 
 build:csp-lmc-mid_pkg:
-  extends: .build_python_release
+  extends: .build_python
+  script:
+    - cd $BUILD_PATH
+    - python3 setup.py egg_info -b+$CI_COMMIT_SHORT_SHA sdist bdist_wheel
   variables:
     BUILD_PATH: csp-lmc-mid
   only: [master]
@@ -51,10 +40,25 @@ build:csp-lmc-mid_image:
   script:
     - cd $BUILD_PATH      
     - make build
-    - make push
   variables:
     BUILD_PATH: csp-lmc-mid
 
+build:csp-lmc-mid_dev-image:
+  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
+  before_script:
+  - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
+  tags:
+    - k8srunner
+  stage: build
+  script:
+    - cd $BUILD_PATH      
+    - echo $IMG
+    - docker build -t $IMG  --build-arg PROJECT_ID=$CI_PROJECT_ID -f Dockerfile.gitlab .
+    - echo -n "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
+    - docker push "$IMG"
+  variables:
+    BUILD_PATH: csp-lmc-mid
+    IMG: "$CI_REGISTRY_IMAGE:mid-csp-lmc-$CI_COMMIT_SHORT_SHA"
 # test:csp-lmc-mid:
 #  stage: test
 #  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
diff --git a/csp-lmc-mid/Dockerfile.gitlab b/csp-lmc-mid/Dockerfile.gitlab
new file mode 100644
index 0000000..ed78116
--- /dev/null
+++ b/csp-lmc-mid/Dockerfile.gitlab
@@ -0,0 +1,16 @@
+FROM nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:9.3.2 AS buildenv
+FROM nexus.engageska-portugal.pt/ska-docker/ska-python-runtime:9.3.2 AS runtime
+
+# create ipython profile to so that itango doesn't fail if ipython hasn't run yet
+RUN ipython profile create
+
+ENV PATH=/home/tango/.local/bin:$PATH
+ARG PROJECT_ID
+# install all ska packages from nexus repository before installing the dev 
+# csp-lmc-common package. In this way the csp-òmc-common dependencies are already
+# in place and the package can be installed from the gitlab repo without errors.
+RUN python3 -m pip install -r requirements-gitlab.txt .
+#install
+RUN python3 -m pip install -U --index-url https://gitlab.com/api/v4/projects/$PROJECT_ID/packages/pypi/simple csp-lmc-common 
+
+CMD ["/venv/bin/python","/app/csp_lmc_mid/MidCspMaster.py"]
diff --git a/csp-lmc-mid/requirements-gitlab.txt b/csp-lmc-mid/requirements-gitlab.txt
new file mode 100644
index 0000000..a603b97
--- /dev/null
+++ b/csp-lmc-mid/requirements-gitlab.txt
@@ -0,0 +1,7 @@
+numpy == 1.17.2
+pytango >= 9.3.2
+jsonschema >= 3.2.0
+lmcbaseclasses >=0.8.1
+ska-log-transactions
+ska-telescope-model
+
-- 
GitLab


From 7b1d861dcac3165b1c25888129c77533d12252ce Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 17:54:59 +0100
Subject: [PATCH 25/46] TestPipeline: fix error in yaml file.

---
 csp-lmc-mid/.gitlab-ci.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 6916eb4..2e9b335 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -14,10 +14,6 @@
     paths:
       - ./$BUILD_PATH/dist/
 
-build:csp-lmc-mid_pkg_dev: # Executed on a commit
-  variables:
-    BUILD_PATH: csp-lmc-mid
-
 build:csp-lmc-mid_pkg:
   extends: .build_python
   script:
-- 
GitLab


From 55b49db0bf366ffb99870c0d2a6f6fef489edade Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 18:08:03 +0100
Subject: [PATCH 26/46] TestPipeline: removed unused stages.

---
 csp-lmc-mid/.gitlab-ci.yml | 69 +++++++-------------------------------
 1 file changed, 12 insertions(+), 57 deletions(-)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 2e9b335..09057a8 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -24,22 +24,12 @@ build:csp-lmc-mid_pkg:
   only: [master]
 
 #
-# Build and push csp-lmc-mid image 
+# Build and push csp-lmc-mid image on the gitlab repository.
+# Use the csp-lmc-common package stored in the gitlab Package Registry.
+# The image is used to run the integration tests running in k8s environment.
 #
-build:csp-lmc-mid_image:
-  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
-  before_script:
-  - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
-  tags:
-    - k8srunner
-  stage: build
-  script:
-    - cd $BUILD_PATH      
-    - make build
-  variables:
-    BUILD_PATH: csp-lmc-mid
 
-build:csp-lmc-mid_dev-image:
+build:csp-lmc-mid_image:
   image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
   before_script:
   - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
@@ -55,22 +45,7 @@ build:csp-lmc-mid_dev-image:
   variables:
     BUILD_PATH: csp-lmc-mid
     IMG: "$CI_REGISTRY_IMAGE:mid-csp-lmc-$CI_COMMIT_SHORT_SHA"
-# test:csp-lmc-mid:
-#  stage: test
-#  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
-#  before_script:
-#      - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
-#      - *install_pip
-#  tags:
-#    - k8srunner
-#  script:
-#    - echo $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
-#    - pip3 install -U $(ls -d ./csp-lmc-mid/dist/*.whl | grep $CI_COMMIT_SHORT_SHA) --extra-index-url https://nexus.engageska-portugal.pt/repository/pypi/simple
-#    - cd csp-lmc-mid/docker
-#    - make test
-#  artifacts:
-#    paths:
-#      - build
+
 
 #  
 # Test csp-lmc-mid project
@@ -145,8 +120,7 @@ linting:csp-lmc-mid:
 #
 # Publish csp-lmc-mid python package
 #
-
-.publish_python:
+publish:csp-lmc-mid:
   stage: publish
   image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
   variables:
@@ -156,36 +130,15 @@ linting:csp-lmc-mid:
     - k8srunner
   before_script:
     - pip install twine
-
-.publish_python_dev:
-  extends: .publish_python
-  except: [master]
-  script:
-    - cd $BUILD_PATH
-    - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true
-  when: manual
-
-.publish_python_release:
-  extends: .publish_python
-  only: [master]
+  dependencies:
+    - build:csp-lmc-mid_pkg
   script:
     - cd $BUILD_PATH
     - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true
     - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true
-
-publish:csp-lmc-mid_dev:
-  extends: .publish_python_dev
-  dependencies:
-    - build:csp-lmc-mid_pkg_dev
-  variables:
-    BUILD_PATH: csp-lmc-mid
-
-publish:csp-lmc-mid:
-  extends: .publish_python_release
-  dependencies:
-    - build:csp-lmc-mid_pkg
   variables:
     BUILD_PATH: csp-lmc-mid
+  only: [master]
 
 publish_chart:
   variables:
@@ -203,8 +156,9 @@ publish_chart:
     - curl -s https://gitlab.com/ska-telescope/stupid/raw/master/scripts/publish-charts.sh | bash
   variables:
     BUILD_PATH: csp-lmc-mid
+
 #
-# Release csp-lmc-mid image on nexus
+# Build and release csp-lmc-mid image on nexus
 #
 release csp-lmc-mid image:
   tags:
@@ -214,6 +168,7 @@ release csp-lmc-mid image:
     - build:csp-lmc-mid_image
   script:
     - cd $BUILD_PATH      
+    - make build
     - make push
   when: manual
   only:
-- 
GitLab


From c0e2b0e94a9b74426d10ba9e263a30b8a2b3954e Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 18:19:31 +0100
Subject: [PATCH 27/46] TestPipeline: remove ska-log-transactions from setup.py
 because dependencies from ska-skuid not found.

---
 csp-lmc-common/requirements.txt | 2 +-
 csp-lmc-common/setup.py         | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/csp-lmc-common/requirements.txt b/csp-lmc-common/requirements.txt
index b157404..558ef5c 100644
--- a/csp-lmc-common/requirements.txt
+++ b/csp-lmc-common/requirements.txt
@@ -1,6 +1,6 @@
 numpy == 1.17.2
 pytango >= 9.3.2
 jsonschema >= 3.2.0
-lmcbaseclasses >= 0.7.2
+lmcbaseclasses >= 0.8.1
 ska-log-transactions
 ska-telescope-model
diff --git a/csp-lmc-common/setup.py b/csp-lmc-common/setup.py
index b7ff480..b9315af 100644
--- a/csp-lmc-common/setup.py
+++ b/csp-lmc-common/setup.py
@@ -40,8 +40,7 @@ setup(
             'pytango >9.3.1',
             'future',
             'lmcbaseclasses',
-            'ska-telescope-model',
-            'ska-log-transactions'
+            'ska-telescope-model'
         ],
         dependency_links=[
             'https://nexus.engageska-portugal.pt/repository/pypi/simple'
-- 
GitLab


From 2ea466ae46dc1a5d056ae49a1f3959e425f4e4e0 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 22:27:45 +0100
Subject: [PATCH 28/46] TestPipeline: Try to use the image in gitlab Container
 Registry to deploy the system to run integration tests.

---
 csp-lmc-mid/.gitlab-ci.yml | 2 +-
 csp-lmc-mid/.make/k8s.mk   | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 09057a8..2eafa28 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -44,7 +44,7 @@ build:csp-lmc-mid_image:
     - docker push "$IMG"
   variables:
     BUILD_PATH: csp-lmc-mid
-    IMG: "$CI_REGISTRY_IMAGE:mid-csp-lmc-$CI_COMMIT_SHORT_SHA"
+    IMG: "mid-csp-lmc:$CI_COMMIT_SHORT_SHA"
 
 
 #  
diff --git a/csp-lmc-mid/.make/k8s.mk b/csp-lmc-mid/.make/k8s.mk
index 2d88343..2d12d51 100644
--- a/csp-lmc-mid/.make/k8s.mk
+++ b/csp-lmc-mid/.make/k8s.mk
@@ -12,6 +12,10 @@ CI_ENVIRONMENT_SLUG ?= mid-csp
 
 # in release.mk it's defined the value of the variable IMAGE_TAG
 SET_IMAGE_TAG ?= --set mid-csp.midcsplmc.image.tag=$(IMAGE_TAG) 
+ifneq ($(CI_JOB_ID),)
+CI_PROJECT_IMAGE :=
+SET_IMAGE_TAG = --set mid-csp.midcsplmc.image.registry=$(CI_REGISTRY)/ska-telescope \
+        --set mid-csp.midcsplmc.image.tag=$(CI_COMMIT_SHORT_SHA)
 .DEFAULT_GOAL := help
 
 k8s: ## Which kubernetes are we connected to
-- 
GitLab


From 8fe2da5d98ebc79e68b698e7a57e4d7e5dc65dde Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 22:39:27 +0100
Subject: [PATCH 29/46] TestPipeline: fix image name

---
 csp-lmc-mid/.gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 2eafa28..4982a79 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -44,7 +44,7 @@ build:csp-lmc-mid_image:
     - docker push "$IMG"
   variables:
     BUILD_PATH: csp-lmc-mid
-    IMG: "mid-csp-lmc:$CI_COMMIT_SHORT_SHA"
+    IMG: "$CI_REGISTRY/ska-telescope/mid-csp-lmc:$CI_COMMIT_SHORT_SHA"
 
 
 #  
-- 
GitLab


From fe816644230f2f1f27ed119fbbddff49662f37a4 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 23:00:31 +0100
Subject: [PATCH 30/46] TestPipeline: In gitlab repo the image name has to be
 equal to the repository name (csp-lmc). Use --set option to override the
 image name in k8s (for test)

---
 csp-lmc-mid/.gitlab-ci.yml | 2 +-
 csp-lmc-mid/.make/k8s.mk   | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 4982a79..3b3b297 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -44,7 +44,7 @@ build:csp-lmc-mid_image:
     - docker push "$IMG"
   variables:
     BUILD_PATH: csp-lmc-mid
-    IMG: "$CI_REGISTRY/ska-telescope/mid-csp-lmc:$CI_COMMIT_SHORT_SHA"
+    IMG: "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA"
 
 
 #  
diff --git a/csp-lmc-mid/.make/k8s.mk b/csp-lmc-mid/.make/k8s.mk
index 2d12d51..e54857c 100644
--- a/csp-lmc-mid/.make/k8s.mk
+++ b/csp-lmc-mid/.make/k8s.mk
@@ -15,6 +15,7 @@ SET_IMAGE_TAG ?= --set mid-csp.midcsplmc.image.tag=$(IMAGE_TAG)
 ifneq ($(CI_JOB_ID),)
 CI_PROJECT_IMAGE :=
 SET_IMAGE_TAG = --set mid-csp.midcsplmc.image.registry=$(CI_REGISTRY)/ska-telescope \
+        --set mid-csp.midcsplmc.image=csp-lmc \
         --set mid-csp.midcsplmc.image.tag=$(CI_COMMIT_SHORT_SHA)
 .DEFAULT_GOAL := help
 
-- 
GitLab


From 05d1bf44501edd02e7a7a62c4eaadbfae89cef3e Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 23:16:55 +0100
Subject: [PATCH 31/46] TestPipeline: fix bug in makefile

---
 csp-lmc-mid/.make/k8s.mk | 1 +
 1 file changed, 1 insertion(+)

diff --git a/csp-lmc-mid/.make/k8s.mk b/csp-lmc-mid/.make/k8s.mk
index e54857c..192b886 100644
--- a/csp-lmc-mid/.make/k8s.mk
+++ b/csp-lmc-mid/.make/k8s.mk
@@ -17,6 +17,7 @@ CI_PROJECT_IMAGE :=
 SET_IMAGE_TAG = --set mid-csp.midcsplmc.image.registry=$(CI_REGISTRY)/ska-telescope \
         --set mid-csp.midcsplmc.image=csp-lmc \
         --set mid-csp.midcsplmc.image.tag=$(CI_COMMIT_SHORT_SHA)
+endif
 .DEFAULT_GOAL := help
 
 k8s: ## Which kubernetes are we connected to
-- 
GitLab


From 242cf57ec556da06a1abb5062344d399e66108f0 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Tue, 12 Jan 2021 23:50:20 +0100
Subject: [PATCH 32/46] TestPipeline: try to fix problems in deploying the
 system for test.

---
 csp-lmc-mid/.gitlab-ci.yml | 4 ++++
 csp-lmc-mid/.make/k8s.mk   | 3 ++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 3b3b297..8785b30 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -83,6 +83,10 @@ integration:csp-lmc-mid:
   script:
     - export KUBE_NAMESPACE="ci-csp-proto-$CI_COMMIT_SHORT_SHA"
     - cd $BUILD_PATH  
+    - make k8s
+    - make install-chart
+    - make template-chart
+    - make wait
     - make test
   environment:
       name: test
diff --git a/csp-lmc-mid/.make/k8s.mk b/csp-lmc-mid/.make/k8s.mk
index 192b886..978879f 100644
--- a/csp-lmc-mid/.make/k8s.mk
+++ b/csp-lmc-mid/.make/k8s.mk
@@ -17,6 +17,7 @@ CI_PROJECT_IMAGE :=
 SET_IMAGE_TAG = --set mid-csp.midcsplmc.image.registry=$(CI_REGISTRY)/ska-telescope \
         --set mid-csp.midcsplmc.image=csp-lmc \
         --set mid-csp.midcsplmc.image.tag=$(CI_COMMIT_SHORT_SHA)
+IMAGE_TO_TEST = $(CI_REGISTRY_IMAGE)/$(CI_COMMIT_SHORT_SHA)
 endif
 .DEFAULT_GOAL := help
 
@@ -213,7 +214,7 @@ kubeconfig: ## export current KUBECONFIG as base64 ready for KUBE_CONFIG_BASE64
 # base64 payload is given a boundary "~~~~BOUNDARY~~~~" and extracted using perl
 # clean up the run to completion container
 # exit the saved status
-test: install-chart wait ## test the application on K8s
+test:  ## test the application on K8s
 	$(call k8s_test,test); \
 		status=$$?; \
 		rm -rf build; \
-- 
GitLab


From aec4f053eb8aa20c97e8ae71f91a6a2ef2fddae8 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Wed, 13 Jan 2021 00:23:32 +0100
Subject: [PATCH 33/46] TestPipeline: fix bug in setting helm options.

---
 csp-lmc-mid/.make/k8s.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/csp-lmc-mid/.make/k8s.mk b/csp-lmc-mid/.make/k8s.mk
index 978879f..c49aa84 100644
--- a/csp-lmc-mid/.make/k8s.mk
+++ b/csp-lmc-mid/.make/k8s.mk
@@ -15,7 +15,7 @@ SET_IMAGE_TAG ?= --set mid-csp.midcsplmc.image.tag=$(IMAGE_TAG)
 ifneq ($(CI_JOB_ID),)
 CI_PROJECT_IMAGE :=
 SET_IMAGE_TAG = --set mid-csp.midcsplmc.image.registry=$(CI_REGISTRY)/ska-telescope \
-        --set mid-csp.midcsplmc.image=csp-lmc \
+        --set mid-csp.midcsplmc.image.image=csp-lmc \
         --set mid-csp.midcsplmc.image.tag=$(CI_COMMIT_SHORT_SHA)
 IMAGE_TO_TEST = $(CI_REGISTRY_IMAGE)/$(CI_COMMIT_SHORT_SHA)
 endif
-- 
GitLab


From 0910e94c315eed19a8ec3a04c31e3e15d5cfe7a8 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Wed, 13 Jan 2021 00:46:56 +0100
Subject: [PATCH 34/46] TestPipeline: uninstall the helm chart before running
 tests.

---
 csp-lmc-mid/.gitlab-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 8785b30..18c06d9 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -84,6 +84,7 @@ integration:csp-lmc-mid:
     - export KUBE_NAMESPACE="ci-csp-proto-$CI_COMMIT_SHORT_SHA"
     - cd $BUILD_PATH  
     - make k8s
+    - make uninstall-chart
     - make install-chart
     - make template-chart
     - make wait
-- 
GitLab


From dc52206f3b1add9c4d14d9870d2d9c95439b44eb Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Wed, 13 Jan 2021 00:57:10 +0100
Subject: [PATCH 35/46] TestPipeline: removed uninstall chart because it causes
 pipeline error.

---
 csp-lmc-mid/.gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 18c06d9..8785b30 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -84,7 +84,6 @@ integration:csp-lmc-mid:
     - export KUBE_NAMESPACE="ci-csp-proto-$CI_COMMIT_SHORT_SHA"
     - cd $BUILD_PATH  
     - make k8s
-    - make uninstall-chart
     - make install-chart
     - make template-chart
     - make wait
-- 
GitLab


From 4dce09c69fc61ad67f70e2c0ebb96f16a1432fea Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Wed, 13 Jan 2021 11:39:27 +0100
Subject: [PATCH 36/46] TestPipeline: fix error in image name.

---
 csp-lmc-mid/.make/k8s.mk      | 2 +-
 csp-lmc-mid/Dockerfile.gitlab | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/csp-lmc-mid/.make/k8s.mk b/csp-lmc-mid/.make/k8s.mk
index c49aa84..c3eb95b 100644
--- a/csp-lmc-mid/.make/k8s.mk
+++ b/csp-lmc-mid/.make/k8s.mk
@@ -17,7 +17,7 @@ CI_PROJECT_IMAGE :=
 SET_IMAGE_TAG = --set mid-csp.midcsplmc.image.registry=$(CI_REGISTRY)/ska-telescope \
         --set mid-csp.midcsplmc.image.image=csp-lmc \
         --set mid-csp.midcsplmc.image.tag=$(CI_COMMIT_SHORT_SHA)
-IMAGE_TO_TEST = $(CI_REGISTRY_IMAGE)/$(CI_COMMIT_SHORT_SHA)
+IMAGE_TO_TEST = $(CI_REGISTRY_IMAGE):$(CI_COMMIT_SHORT_SHA)
 endif
 .DEFAULT_GOAL := help
 
diff --git a/csp-lmc-mid/Dockerfile.gitlab b/csp-lmc-mid/Dockerfile.gitlab
index ed78116..95fac14 100644
--- a/csp-lmc-mid/Dockerfile.gitlab
+++ b/csp-lmc-mid/Dockerfile.gitlab
@@ -1,3 +1,8 @@
+# Dockerfile used during CI/CD pipeline execution.
+# The docker image is built using the csp-lmc-common python
+# package uploaded to the gitlab Package Repository.
+# PROJECT_ID: is the csp-lmc project ID under Gitlab: the value is passed as argument
+# during the image build stage.
 FROM nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:9.3.2 AS buildenv
 FROM nexus.engageska-portugal.pt/ska-docker/ska-python-runtime:9.3.2 AS runtime
 
@@ -7,7 +12,7 @@ RUN ipython profile create
 ENV PATH=/home/tango/.local/bin:$PATH
 ARG PROJECT_ID
 # install all ska packages from nexus repository before installing the dev 
-# csp-lmc-common package. In this way the csp-òmc-common dependencies are already
+# csp-lmc-common package. In this way the csp-lmc-common dependencies are already
 # in place and the package can be installed from the gitlab repo without errors.
 RUN python3 -m pip install -r requirements-gitlab.txt .
 #install
-- 
GitLab


From 8541d77951d31278deba70333263b52f865db045 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 13:05:44 +0100
Subject: [PATCH 37/46] TestPipeline: try to use the correct common-lmc-package
 version built during the build stage.

---
 csp-lmc-common/.gitlab-ci.yml | 1 +
 csp-lmc-mid/.gitlab-ci.yml    | 7 ++++++-
 csp-lmc-mid/Dockerfile.gitlab | 4 +++-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 469c1a1..bfab5c9 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -134,6 +134,7 @@ linting:csp-lmc-common:
   script:
     - cd $BUILD_PATH
     - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
+
 .publish_python_release_common:
   extends: .publish_python_common
   only: [master]
diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index 8785b30..f36a73e 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -33,17 +33,22 @@ build:csp-lmc-mid_image:
   image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
   before_script:
   - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
+  - cd $COMMON_PATH      
+  - export COMMMON_VER=$(python setup.py --version)
+  - echo $COMMON_VER
+  - cd ..
   tags:
     - k8srunner
   stage: build
   script:
     - cd $BUILD_PATH      
     - echo $IMG
-    - docker build -t $IMG  --build-arg PROJECT_ID=$CI_PROJECT_ID -f Dockerfile.gitlab .
+    - docker build -t $IMG  --build-arg PROJECT_ID=$CI_PROJECT_ID --build-arg PKG_VER=$COMMON_VER --build-arg RELEASE="+dev.$CI_COMMIT_SHORT_SHA" -f Dockerfile.gitlab .
     - echo -n "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" --password-stdin $CI_REGISTRY
     - docker push "$IMG"
   variables:
     BUILD_PATH: csp-lmc-mid
+    COMMON_PATH: csp-lmc-common
     IMG: "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA"
 
 
diff --git a/csp-lmc-mid/Dockerfile.gitlab b/csp-lmc-mid/Dockerfile.gitlab
index 95fac14..b17ac41 100644
--- a/csp-lmc-mid/Dockerfile.gitlab
+++ b/csp-lmc-mid/Dockerfile.gitlab
@@ -11,11 +11,13 @@ RUN ipython profile create
 
 ENV PATH=/home/tango/.local/bin:$PATH
 ARG PROJECT_ID
+ARG RELEASE
+ARG PKG_VER 
 # install all ska packages from nexus repository before installing the dev 
 # csp-lmc-common package. In this way the csp-lmc-common dependencies are already
 # in place and the package can be installed from the gitlab repo without errors.
 RUN python3 -m pip install -r requirements-gitlab.txt .
 #install
-RUN python3 -m pip install -U --index-url https://gitlab.com/api/v4/projects/$PROJECT_ID/packages/pypi/simple csp-lmc-common 
+RUN python3 -m pip install -U --index-url https://gitlab.com/api/v4/projects/$PROJECT_ID/packages/pypi/simple csp-lmc-common=="$PKG_VER$RELEASE"
 
 CMD ["/venv/bin/python","/app/csp_lmc_mid/MidCspMaster.py"]
-- 
GitLab


From 16c3c1a17e123f756276cd48df9ff4c4c42919f3 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 14:13:58 +0100
Subject: [PATCH 38/46] TestPipeline: removed csp-lmc-common image build.
 Specify ska-telescope-model version (0.2.0)

---
 csp-lmc-common/.gitlab-ci.yml   | 56 ++++++++++++++++-----------------
 csp-lmc-common/requirements.txt |  2 +-
 2 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index bfab5c9..bd698be 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -42,19 +42,19 @@ build:csp-lmc-common_pkg:
 #
 # Build csp-lmc-common image 
 #
-build:csp-lmc-common_image:
-  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
-  before_script:
-  - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
-  tags:
-    - k8srunner
-    - engageska
-  stage: build_common
-  script:
-    - cd $COMMON_BUILD_PATH      
-    - make build
-  variables:
-    COMMON_BUILD_PATH: csp-lmc-common/docker
+#build:csp-lmc-common_image:
+#  image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
+#  before_script:
+#  - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
+#  tags:
+#    - k8srunner
+#    - engageska
+#  stage: build_common
+#  script:
+#    - cd $COMMON_BUILD_PATH      
+#    - make build
+#  variables:
+#    COMMON_BUILD_PATH: csp-lmc-common/docker
 
 #  
 # Test csp-lmc-common project
@@ -160,18 +160,18 @@ publish:csp-lmc-common:
 #
 # Release csp-lmc-common image on nexus
 #
-release csp-lmc-common image:
-  tags:
-    - k8srunner
-  stage: release
-  dependencies:
-    - build:csp-lmc-common_image
-  script:
-    - cd $COMMON_BUILD_PATH      
-    - make push
-  when: manual
-  only:
-    refs:
-      - master
-  variables:
-    COMMON_BUILD_PATH: csp-lmc-common/docker
+#release csp-lmc-common image:
+#  tags:
+#    - k8srunner
+#  stage: release
+#  dependencies:
+#    - build:csp-lmc-common_image
+#  script:
+#    - cd $COMMON_BUILD_PATH      
+#    - make push
+#  when: manual
+#  only:
+#    refs:
+#      - master
+#  variables:
+#    COMMON_BUILD_PATH: csp-lmc-common/docker
diff --git a/csp-lmc-common/requirements.txt b/csp-lmc-common/requirements.txt
index 558ef5c..6f987de 100644
--- a/csp-lmc-common/requirements.txt
+++ b/csp-lmc-common/requirements.txt
@@ -3,4 +3,4 @@ pytango >= 9.3.2
 jsonschema >= 3.2.0
 lmcbaseclasses >= 0.8.1
 ska-log-transactions
-ska-telescope-model
+ska-telescope-model==0.2.0
-- 
GitLab


From 417a4a5c5f6a4c0f489b0e5192de81a919cb27d8 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 14:26:51 +0100
Subject: [PATCH 39/46] TestPipeline: Specify ska-telescope-model version equal
 to 0.2.0 in requirements.txt

---
 csp-lmc-mid/.gitlab-ci.yml   | 2 +-
 csp-lmc-mid/requirements.txt | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index f36a73e..d6b7cc0 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -34,7 +34,7 @@ build:csp-lmc-mid_image:
   before_script:
   - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
   - cd $COMMON_PATH      
-  - export COMMMON_VER=$(python setup.py --version)
+  - export COMMMON_VER=$(python3 setup.py --version)
   - echo $COMMON_VER
   - cd ..
   tags:
diff --git a/csp-lmc-mid/requirements.txt b/csp-lmc-mid/requirements.txt
index 9f4258b..f807994 100644
--- a/csp-lmc-mid/requirements.txt
+++ b/csp-lmc-mid/requirements.txt
@@ -1,8 +1,8 @@
 numpy == 1.17.2
 pytango >= 9.3.2
 jsonschema >= 3.2.0
-lmcbaseclasses >= 0.7.2
+lmcbaseclasses >= 0.8.0
 csp-lmc-common >= 0.6.12
 ska-log-transactions
-ska-telescope-model
+ska-telescope-model==0.2.0
 
-- 
GitLab


From 3ef07b60514a88fc152585e2d1bf9a86aed7205f Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 14:54:29 +0100
Subject: [PATCH 40/46] TestPipeline: publish packages and images on nexus repo
 only on tags.

---
 csp-lmc-common/.gitlab-ci.yml |  4 +++-
 csp-lmc-mid/.gitlab-ci.yml    | 10 ++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index bd698be..3dd961a 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -37,7 +37,9 @@ build:csp-lmc-common_pkg:
   extends: .build_python_release_common
   variables:
     COMMON_BUILD_PATH: csp-lmc-common
-  only: [master]
+  only: 
+    refs:
+      - tags
 
 #
 # Build csp-lmc-common image 
diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index d6b7cc0..b279c7a 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -34,7 +34,7 @@ build:csp-lmc-mid_image:
   before_script:
   - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
   - cd $COMMON_PATH      
-  - export COMMMON_VER=$(python3 setup.py --version)
+  - export COMMON_VER=$(python3 setup.py --version)
   - echo $COMMON_VER
   - cd ..
   tags:
@@ -147,7 +147,9 @@ publish:csp-lmc-mid:
     - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true
   variables:
     BUILD_PATH: csp-lmc-mid
-  only: [master]
+  only: 
+    refs:
+      - tags
 
 publish_chart:
   variables:
@@ -179,9 +181,9 @@ release csp-lmc-mid image:
     - cd $BUILD_PATH      
     - make build
     - make push
-  when: manual
+  when: on_success
   only:
     refs:
-      - master
+      - tags
   variables:
     BUILD_PATH: csp-lmc-mid
-- 
GitLab


From b614805d396b9b43cdfd9d4fe99f5077cacd3168 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 15:31:20 +0100
Subject: [PATCH 41/46] TestPipeline: Specify version ska-telescope-model 0.2.0
 in setup.py to installl the csp-lmc-common package.

---
 csp-lmc-common/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/csp-lmc-common/setup.py b/csp-lmc-common/setup.py
index b9315af..ca547d0 100644
--- a/csp-lmc-common/setup.py
+++ b/csp-lmc-common/setup.py
@@ -40,7 +40,7 @@ setup(
             'pytango >9.3.1',
             'future',
             'lmcbaseclasses',
-            'ska-telescope-model'
+            'ska-telescope-model==0.2.0'
         ],
         dependency_links=[
             'https://nexus.engageska-portugal.pt/repository/pypi/simple'
-- 
GitLab


From 93b525f26036eb4e5193c4284237589718041824 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 15:41:43 +0100
Subject: [PATCH 42/46] TestPipeline: specify version of ska-telescope-model
 package also for CI/CD pipelines execution

---
 csp-lmc-mid/requirements-gitlab.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/csp-lmc-mid/requirements-gitlab.txt b/csp-lmc-mid/requirements-gitlab.txt
index a603b97..6bcd9f2 100644
--- a/csp-lmc-mid/requirements-gitlab.txt
+++ b/csp-lmc-mid/requirements-gitlab.txt
@@ -3,5 +3,5 @@ pytango >= 9.3.2
 jsonschema >= 3.2.0
 lmcbaseclasses >=0.8.1
 ska-log-transactions
-ska-telescope-model
+ska-telescope-model==0.2.0
 
-- 
GitLab


From e842931b53b27a29180eca3ba55a205793868d8a Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 17:15:56 +0100
Subject: [PATCH 43/46] TestPipeline: cleanup common pipeline file.

---
 .gitlab-ci.yml                |  1 +
 csp-lmc-common/.gitlab-ci.yml | 88 ++++++++++++++++-------------------
 2 files changed, 42 insertions(+), 47 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f8af453..087474f 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -93,6 +93,7 @@ pages:
     #- coverage xml -i
     # rename file because bagdges creation looks for code-coverage.xml file
     #- mv coverage.xml ./reports/code-coverage.xml
+  allow_failure: true
   artifacts:
     paths:
       - public 
diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 3dd961a..1597c4e 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -6,7 +6,6 @@
 
 .build_python_common:
   stage: build_common
-  dependencies: []
   image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
   tags:
     - k8srunner
@@ -14,32 +13,26 @@
   artifacts:
     paths:
       - ./$COMMON_BUILD_PATH/dist/
+  variables:
+    COMMON_BUILD_PATH: csp-lmc-common
+
 
-.build_python_dev_common:
+build:csp-lmc-common_pkg_dev: # Executed on a commit
   extends: .build_python_common
-  dependencies: []
   script:
     - cd $COMMON_BUILD_PATH
     - python3 setup.py egg_info -b+dev.$CI_COMMIT_SHORT_SHA sdist bdist_wheel
+  except:
+    - master
 
-.build_python_release_common:
+build:csp-lmc-common_pkg:
   extends: .build_python_common
   script:
     - cd $COMMON_BUILD_PATH
     - python3 setup.py egg_info -b+$CI_COMMIT_SHORT_SHA sdist bdist_wheel
-
-build:csp-lmc-common_pkg_dev: # Executed on a commit
-  extends: .build_python_dev_common
-  variables:
-    COMMON_BUILD_PATH: csp-lmc-common
-
-build:csp-lmc-common_pkg:
-  extends: .build_python_release_common
-  variables:
-    COMMON_BUILD_PATH: csp-lmc-common
-  only: 
-    refs:
+  only:
       - tags
+      - master
 
 #
 # Build csp-lmc-common image 
@@ -62,19 +55,11 @@ build:csp-lmc-common_pkg:
 # Test csp-lmc-common project
 #
 
-.test_common:
+test:csp-lmc-common:
   image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
+  stage: test_common
   tags:
     - k8srunner
-  artifacts:
-    paths:
-      - ./$COMMON_BUILD_PATH/build/
-  variables:
-    COMMON_BUILD_PATH: csp-lmc-common
-
-test:csp-lmc-common:
-  extends: .test_common
-  stage: test_common
   script:
     - echo $CI_REGISTRY_IMAGE
     - echo $CI_REGISTRY
@@ -90,6 +75,11 @@ test:csp-lmc-common:
     - mv setup_py_test.stdout build/csp-lmc-common-setup-test.stdout
     - mv htmlcov build/csp-lmc-common_htmlcov
     - mv coverage.xml build
+  artifacts:
+    paths:
+      - ./$COMMON_BUILD_PATH/build/
+  variables:
+    COMMON_BUILD_PATH: csp-lmc-common
 #
 # linting stage
 
@@ -122,42 +112,46 @@ linting:csp-lmc-common:
 .publish_python_common:
   stage: publish_common
   image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
-  variables:
-    TWINE_USERNAME: $TWINE_USERNAME
-    TWINE_PASSWORD: $TWINE_PASSWORD
   tags:
     - k8srunner
   before_script:
     - python3 -m pip install twine
+  variables:
+    COMMON_BUILD_PATH: csp-lmc-common
 
-.publish_python_dev_common:
-  extends: .publish_python_common
-  except: [master]
-  script:
-    - cd $BUILD_PATH
-    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
-
-.publish_python_release_common:
-  extends: .publish_python_common
-  only: [master]
-  script:
-    - cd $COMMON_BUILD_PATH
-    - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true
-    - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true
-
+# upload the common package to the Gitlab Package Repository.
+# Se the '--skip-existing' because if the CI/CD pipeline is re-run (without doing a commit)
+# the package version does not change and the upload to the repository fails.      
 publish:csp-lmc-common_dev:
-  extends: .publish_python_dev_common
+  extends: .publish_python_common
   dependencies:
     - build:csp-lmc-common_pkg_dev
   variables:
     BUILD_PATH: csp-lmc-common
+  variables:
+    TWINE_USERNAME: gitlab-ci-token
+    TWINE_PASSWORD: $CI_JOB_TOKEN
+  script:
+    - cd $COMMON_BUILD_PATH
+    - python3 -m twine upload --skip-existing --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
 
+# upload the package to the Nexus Repository only on project tagging.
 publish:csp-lmc-common:
-  extends: .publish_python_release_common
+  extends: .publish_python_common
   dependencies:
     - build:csp-lmc-common_pkg
   variables:
-    COMMON_BUILD_PATH: csp-lmc-common
+    TWINE_USERNAME: $TWINE_USERNAME
+    TWINE_PASSWORD: $TWINE_PASSWORD
+  only:
+    refs:
+      - tags
+  script:
+    - cd $COMMON_BUILD_PATH
+    - twine upload --repository-url $PYPI_REPOSITORY_URL dist/* || true
+    - twine upload --skip-existing -u $PYPI_USER -p $PYPI_PASS dist/* || true
+
+
 
 #
 # Release csp-lmc-common image on nexus
-- 
GitLab


From ae6f5ba5ab21d463e7ae5c420008d3548343cada Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 17:24:15 +0100
Subject: [PATCH 44/46] TestPipeline: pass user and pwd directly to twine (to
 upload the package)

---
 csp-lmc-common/.gitlab-ci.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/csp-lmc-common/.gitlab-ci.yml b/csp-lmc-common/.gitlab-ci.yml
index 1597c4e..a2eeb6e 100644
--- a/csp-lmc-common/.gitlab-ci.yml
+++ b/csp-lmc-common/.gitlab-ci.yml
@@ -128,12 +128,9 @@ publish:csp-lmc-common_dev:
     - build:csp-lmc-common_pkg_dev
   variables:
     BUILD_PATH: csp-lmc-common
-  variables:
-    TWINE_USERNAME: gitlab-ci-token
-    TWINE_PASSWORD: $CI_JOB_TOKEN
   script:
     - cd $COMMON_BUILD_PATH
-    - python3 -m twine upload --skip-existing --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python3 -m twine upload --skip-existing --repository-url https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi $(ls -d ./dist/*.whl | grep $CI_COMMIT_SHORT_SHA)
 
 # upload the package to the Nexus Repository only on project tagging.
 publish:csp-lmc-common:
-- 
GitLab


From 105da686a858d356627b83eed7909d795ba47c87 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 18:01:13 +0100
Subject: [PATCH 45/46] CT-220: Updated mid-csp charts to: - image 0.6.10 -
 tango-util 0.2.8 - tango-db 0.2.12 Build and publish mid-csp python package
 only on tags.

---
 csp-lmc-mid/HISTORY                            | 1 +
 csp-lmc-mid/charts/mid-csp-umbrella/Chart.yaml | 4 ++--
 csp-lmc-mid/charts/mid-csp/Chart.yaml          | 8 ++++----
 csp-lmc-mid/charts/mid-csp/values.yaml         | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/csp-lmc-mid/HISTORY b/csp-lmc-mid/HISTORY
index 428141f..db30407 100644
--- a/csp-lmc-mid/HISTORY
+++ b/csp-lmc-mid/HISTORY
@@ -1,6 +1,7 @@
 0.6.10
 - use csp-lmc-common 0.6.12
 - support to ADR-18/22
+- use ska-telescope-model 0.2.0
 
 0.6.9:
 - use csp-lmc-common 0.6.9
diff --git a/csp-lmc-mid/charts/mid-csp-umbrella/Chart.yaml b/csp-lmc-mid/charts/mid-csp-umbrella/Chart.yaml
index f99d236..7ee7885 100644
--- a/csp-lmc-mid/charts/mid-csp-umbrella/Chart.yaml
+++ b/csp-lmc-mid/charts/mid-csp-umbrella/Chart.yaml
@@ -6,7 +6,7 @@ appVersion: 0.6.9
 icon: https://www.skatelescope.org/wp-content/uploads/2016/07/09545_NEW_LOGO_2014.png
 dependencies:
 - name: tango-base
-  version: 0.2.7
+  version: 0.2.12
   repository: https://nexus.engageska-portugal.pt/repository/helm-chart
 - name: mid-cbf
   version: 0.1.1
@@ -15,5 +15,5 @@ dependencies:
   version: 0.1.1
   repository: https://nexus.engageska-portugal.pt/repository/helm-chart
 - name: mid-csp
-  version: 0.1.0
+  version: 0.1.1
   repository: "file://../mid-csp"
diff --git a/csp-lmc-mid/charts/mid-csp/Chart.yaml b/csp-lmc-mid/charts/mid-csp/Chart.yaml
index ccd8ef2..63f8119 100644
--- a/csp-lmc-mid/charts/mid-csp/Chart.yaml
+++ b/csp-lmc-mid/charts/mid-csp/Chart.yaml
@@ -1,15 +1,15 @@
 apiVersion: v2
 name: mid-csp
 description: A Helm chart for deploying the Mid_CSP.LMC devices on Kubernetes
-version: 0.1.0
-appVersion: "0.6.9"
+version: 0.1.1
+appVersion: "0.6.10"
 icon: https://www.skatelescope.org/wp-content/uploads/2016/07/09545_NEW_LOGO_2014.png
 dependencies:
 - name: tango-util
-  version: 0.2.4
+  version: 0.2.8
   repository: https://nexus.engageska-portugal.pt/repository/helm-chart
 - name: tango-base
-  version: 0.2.7
+  version: 0.2.12
   repository: https://nexus.engageska-portugal.pt/repository/helm-chart
   condition: tango-base.enabled,global.sub-system.tango-base.enabled
 
diff --git a/csp-lmc-mid/charts/mid-csp/values.yaml b/csp-lmc-mid/charts/mid-csp/values.yaml
index 42b5db9..2da0693 100644
--- a/csp-lmc-mid/charts/mid-csp/values.yaml
+++ b/csp-lmc-mid/charts/mid-csp/values.yaml
@@ -37,7 +37,7 @@ midcsplmc:
   image:
     registry: nexus.engageska-portugal.pt/ska-docker
     image: mid-csp-lmc
-    tag: 0.6.9
+    tag: 0.6.10
     pullPolicy: IfNotPresent
 
 resources:
-- 
GitLab


From 5b46ff7acba7d673ee135d2a9d10d2445568f884 Mon Sep 17 00:00:00 2001
From: toor <elisabetta.giani@inaf.it>
Date: Thu, 14 Jan 2021 18:31:23 +0100
Subject: [PATCH 46/46] CT-220: Login to nexuss before pulling the image.

---
 csp-lmc-mid/.gitlab-ci.yml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/csp-lmc-mid/.gitlab-ci.yml b/csp-lmc-mid/.gitlab-ci.yml
index b279c7a..df4da27 100644
--- a/csp-lmc-mid/.gitlab-ci.yml
+++ b/csp-lmc-mid/.gitlab-ci.yml
@@ -3,28 +3,25 @@
 # 
 # Build csp-lmc-mid python packages
 # 
-
-.build_python:
+build:csp-lmc-mid_pkg:
   stage: build
   image: nexus.engageska-portugal.pt/ska-docker/ska-python-buildenv:latest
   dependencies: []
   tags:
     - k8srunner
-  artifacts:
-    paths:
-      - ./$BUILD_PATH/dist/
-
-build:csp-lmc-mid_pkg:
-  extends: .build_python
   script:
     - cd $BUILD_PATH
     - python3 setup.py egg_info -b+$CI_COMMIT_SHORT_SHA sdist bdist_wheel
   variables:
     BUILD_PATH: csp-lmc-mid
-  only: [master]
+  artifacts:
+    paths:
+      - ./$BUILD_PATH/dist/
+  only: 
+    - tags
 
 #
-# Build and push csp-lmc-mid image on the gitlab repository.
+# Build and push csp-lmc-mid image on the gitlab Container Registry.
 # Use the csp-lmc-common package stored in the gitlab Package Registry.
 # The image is used to run the integration tests running in k8s environment.
 #
@@ -32,7 +29,6 @@ build:csp-lmc-mid_pkg:
 build:csp-lmc-mid_image:
   image: nexus.engageska-portugal.pt/ska-docker/tango-builder:latest
   before_script:
-  - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
   - cd $COMMON_PATH      
   - export COMMON_VER=$(python3 setup.py --version)
   - echo $COMMON_VER
@@ -128,6 +124,7 @@ linting:csp-lmc-mid:
 
 #
 # Publish csp-lmc-mid python package
+# This stage runs only for tags.
 #
 publish:csp-lmc-mid:
   stage: publish
@@ -170,8 +167,11 @@ publish_chart:
 
 #
 # Build and release csp-lmc-mid image on nexus
+# This stage runs only for tags.
 #
 release csp-lmc-mid image:
+  before_script:
+  - docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD $DOCKER_REGISTRY_HOST
   tags:
     - k8srunner
   stage: release
-- 
GitLab