Commit 5977b83f authored by Elisabetta Giani's avatar Elisabetta Giani
Browse files

Merge branch 'CT-217' into 'master' 

Resolve CT-217

Resolved dead lock of Abort command during Configure.
Release Helm chart 0.1.4.

See merge request ska-telescope/csp-lmc!17
parents cff6ac80 47b1446a

csp-lmc-common/HISTORY

+5 −0
Original line number Diff line number Diff line 
0.7.3

- Solved dead-lock on Abort Command during Configuration

- Restart Command check the if subelements are EMPTY

- Check failures on GoToIdle execution



0.7.2

- Initialization of CSP Subarray align its state ans obsstate to sub-elements.

csp-lmc-common/csp_lmc_common/CspSubarray.py

+36 −51
Original line number Diff line number Diff line
@@ -811,27 +811,16 @@ class CspSubarray(SKASubarray): 
            target_device._list_dev_completed_task[cmd_name] = []

            target_device._cmd_progress[cmd_name] = 0

            target_device._cmd_duration_measured[cmd_name] = 0

            stop_configuration = False

            # flag to signal when configuration ends on a sub-array sub-component

            device_done = defaultdict(lambda:False)

            # inside the end-less lop check the obsState of each sub-component

            device_list = input_arg[0]

            self.logger.info("Trhead started at {}".format(time.time()))

            while True:

                if target_device._abort_obs_event.is_set():

                    self.logger.info("Received and ABORT request during configuration {}".format(time.time()))

                command_progress = 0

                for device in device_list:

                    self.logger.info("Current device {} obsState is {}".format(device,

                                                                     ObsState(target_device._sc_subarray_obs_state[device]).name))

                    if target_device._abort_obs_event.is_set():

                        dev_successful_state = ObsState.ABORTED

                        cmd_name = 'abort'

                        if not stop_configuration:

                            command_start_time = time.time()

                            stop_configuration = True

                            device_done.clear()

                            target_device._reconfiguring = False

                    if device_done[device] == True:

                        continue

                    # if the sub-component execution flag is no more RUNNING, the command has
@@ -886,8 +875,13 @@ class CspSubarray(SKASubarray): 
                target_device._cmd_progress[cmd_name] = command_progress

                if any(target_device._sc_subarray_obs_state[device] == ObsState.CONFIGURING for device in device_list):

                    #target_device._reconfiguring = False

                    self.logger.info("device {} is in {}: reconfiguring is:{}".format(device, ObsState(target_device._sc_subarray_obs_state[device]).name,

                    self.logger.info("device {} is in {}: reconfiguring is:{}".format(device, 

                                                                                      ObsState(target_device._sc_subarray_obs_state[device]).name,

                                                                                      target_device._reconfiguring))

                # check if the abort flag is set

                if target_device._abort_obs_event.is_set():

                    self.logger.info("Received an ABORT request during configuration {}".format(time.time()))

                    return

                if all(value == True for value in device_done.values()):

                    self.logger.info("All devices have been handled at time {}!".format(time.time()))

                    break
@@ -897,14 +891,12 @@ class CspSubarray(SKASubarray): 
                    target_device._timeout_expired = True

                    self.logger.warning("Device went in timeout during configuration")

                    break

                # TODO:

                # check for abort request.

                time.sleep(0.1)

            # end of the while loop

            # acquire the mutex during the check of configuration success/failure. We don't want

            # to receive an boart during this phase otherwise could happen strange situation

            self.logger.info("GOING To lock mutex at {}".format(time.time()))

            with target_device._mutex_obs_state:

            # to receive an abort during this phase otherwise could happen strange situation

            self.logger.info("Going to lock TangoMonitor at {}".format(time.time()))

            with tango.AutoTangoMonitor(self.target):

                # check for timeout/failure conditions on each sub-component

                if any(target_device._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.TIMEOUT for device in device_list):

                    target_device._timeout_expired = True
@@ -918,27 +910,19 @@ class CspSubarray(SKASubarray): 
                self.logger.info("CspSubarray failure flag:{}".format(target_device._failure_raised))

                self.logger.info("CspSubarray timeout flag:{}".format(target_device._timeout_expired))

                if target_device._abort_obs_event.is_set():

                    if target_device._timeout_expired or target_device._failure_raised:

                        return self.failed()

                    self.logger.info("Abort configure ends with success!! {}".format(time.time()))

                    if all(target_device._sc_subarray_obs_state[fqdn] == ObsState.ABORTED for fqdn in device_list):

                        return target_device.abort_cmd_obj.succeeded()

                    return target_device.abort_cmd_obj.abort_monitoring(device_list)



                if target_device._timeout_expired or target_device._failure_raised:

                    # if failure/timeout found check if the CBF subarray is configured. In

                    # this case the CSP.LMC Subarray obsState is set to READY.

                    if target_device._sc_subarray_obs_state[target_device.CbfSubarray] == ObsState.READY:

                        return self.succeeded()

                    self.logger.info("Configure ends with failure")

                    return self.failed()

                    return

                try:

                    if all(target_device._sc_subarray_obs_state[fqdn] == ObsState.READY for fqdn in device_list):

                        target_device._valid_scan_configuration = input_arg[1]

                        target_device._cmd_duration_measured[cmd_name] = time.time() - command_start_time

                        target_device._cmd_progress[cmd_name] = 100

                        target_device._last_executed_command = cmd_name

                    self.logger.info("Configure ends with success!! {}".format(time.time()))

                        self.logger.info("1 Configure ends with success!! {}".format(time.time()))

                        return self.succeeded()

                except Exception as e:

                    self.logger.error(e)

                self.logger.info("Configure ends with failure!! {}".format(time.time()))

                return self.failed()



        def validate_scan_configuration(self, argin):

            """
@@ -1043,8 +1027,6 @@ class CspSubarray(SKASubarray): 
            stop_scan = False

            # inside the end-less loop check the obsState of each sub-component

            while True:

                #self.logger.info("abort:{}".format(target_device._abort_obs_event.is_set()))

                #self.logger.info("end:{}".format(target_device._end_scan_event.is_set()))

                if target_device._stop_thread[cmd_name]:

                    target_device._stop_thread[cmd_name] = False

                    self.logger.info("STOPPING THE THREAD!!!")
@@ -1209,24 +1191,26 @@ class CspSubarray(SKASubarray): 
            if not any(device._sc_subarray_assigned_fqdn):

                # need to add a check also on PSTBeams belonging to subarray

                device_list = device._sc_subarray_fqdn

            self.logger.info("ABORT do at time {}".format(time.time()))

            try:

                abort_group = tango.Group("AbortGroup")

                for fqdn in device_list:

                    abort_group.add(fqdn)

                answer_id = abort_group.command_inout_asynch("Abort")

                device._abort_obs_event.set() 

                self.logger.info("abort is set? {}".format(device._abort_obs_event.is_set()))

                group_reply = abort_group.command_inout_reply(answer_id)

                '''

                if all ((reply.get_data())[0] == ResultCode.FAILED for reply in group_reply):

                    return (ResultCode.FAILED, "Abort FAILED")

                '''    

                device._abort_obs_event.set() 

                self.logger.info("abort is set? {}".format(device._abort_obs_event.is_set()))

            except (TypeError, Exception) as ex:

                self.logger.info(str(ex))

                self.logger.error("TANGO Group command failed")

                return (ResultCode.FAILED, "Abort command failed")

            message = "Abort command completed STARTED"

            self.logger.info(message)

            if all(device._command_thread[cmd_name].is_alive() == False for cmd_name in device._command_thread.keys()):

            #if all(device._command_thread[cmd_name].is_alive() == False for cmd_name in device._command_thread.keys()):

            device._command_thread['abort'] = threading.Thread(target=self.abort_monitoring,

                                                               name="Thread-Abort",

                                                               args=(device_list,))
@@ -1294,6 +1278,8 @@ class CspSubarray(SKASubarray): 
            # set all READY devices in IDLE

            for fqdn in device_list:

                proxy = device._sc_subarray_proxies[fqdn]

                if device._sc_subarray_obs_state[fqdn] == ObsState.EMPTY:

                    continue

                if device._sc_subarray_obs_state[fqdn] == ObsState.READY:

                    proxy.GoToIdle()

            for fqdn in device_list:
@@ -1372,12 +1358,12 @@ class CspSubarray(SKASubarray): 
    class GoToIdleCommand(SKASubarray.EndCommand):

        def do(self):

            target_device = self.target

            target_device._failure_raised = False

            device_list = target_device._sc_subarray_assigned_fqdn

            if not any(target_device._sc_subarray_assigned_fqdn):

                # need to add a check also on PSTBeams belonging to subarray

                device_list = target_device._sc_subarray_fqdn

            try:

                self.logger.info("Creating group for GoToIdle {}".format(device_list))

                sc_group = tango.Group("GoToIdleGroup")

                for device in device_list:

                    sc_group.add(device)
@@ -1388,12 +1374,13 @@ class CspSubarray(SKASubarray): 
            answers = sc_group.command_inout("GoToIdle")

            for reply in answers:

                if reply.has_failed():

                    target_device._failure_raised = True

                    for err in reply.get_err_stack():

                        self.logger.error("device {}: {}-{}".format(reply.dev_name(), err.desc, err.reason))

                else:

                    (result_code,msg) = reply.get_data()

                    self.logger.error("device {}: {}".format(reply.dev_name(), msg))

            if any(target_device._sc_subarray_obs_state[device]== ObsState.FAULT for device in  device_list):

            if any(target_device._sc_subarray_obs_state[device] == ObsState.FAULT for device in  device_list) or target_device._failure_raised:

                return (ResultCode.FAILED, "GoToIdle Command FAILED")

            return (ResultCode.OK, "GoToIdle command executed OK")
@@ -1586,7 +1573,6 @@ class CspSubarray(SKASubarray): 
                                                                                               evt.cmd_name))

                        if evt.cmd_name.lower() == 'configurescan':

                            self._reconfiguring = False

                            self.logger.info("Subarray is in reconfiguring...")

                    if evt.argout[0] == ResultCode.FAILED:

                        self.logger.info("Failure in Device {} while processing the command {}".format(evt.device.dev_name(),

                                                                                               evt.cmd_name))
@@ -2876,7 +2862,6 @@ class CspSubarray(SKASubarray): 
        :return:'DevVarLongStringArray'

        """

        self.logger.info("CALL ABORT at time {}".format(time.time()))

        with self._mutex_obs_state:

        handler = self.get_command_object("Abort")

        (result_code, message) = handler()

        return [[result_code], [message]]

csp-lmc-common/csp_lmc_common/release.py

+1 −1
Original line number Diff line number Diff line
@@ -10,7 +10,7 @@ 
"""Release information for Python Package"""



name = """csp-lmc-common"""

version = "0.7.2"

version = "0.7.3"

version_info = version.split(".")

description = """SKA CSP.LMC Common Software"""

author = "INAF-OAA"

csp-lmc-mid/.release

+3 −3
Original line number Diff line number Diff line
@@ -9,7 +9,7 @@ 
"""Release information for Python Package"""



name = """MID CSP.LMC"""

version = "0.7.2"

version = "0.7.3"

version_info = version.split(".")

description = """SKA MID CSP.LMC Classes"""

author = "E.G"
@@ -20,5 +20,5 @@ copyright = """""" 


#!!!! UPDATE THE MID-CSP IMAGE VERSION

#!!!! DO BEFORE TAGGING !!!!

release=0.7.2

tag=mid-csp-lmc-0.7.2
 
release=0.7.3

tag=mid-csp-lmc-0.7.3

csp-lmc-mid/HISTORY

+5 −1
Original line number Diff line number Diff line 
0.7.3

-use csp-lmc-common 0.7.3

!!! Release helm chart 0.1.4 !!!



0.7.2

- use csp-lmc-common 0.7.2

- updated integration tests for init command
@@ -13,7 +17,7 @@ 
  implementing Assign/ReleaseResources commands.

- added transaction ID for ReleaseResources command.

- updated tests to handle the new commands.

- updated helm chart to version 0.1.2 (mid-csp-lmc:1.0.0 docker image)

- updated helm chart to version 0.1.2 (mid-csp-lmc:0.7.0 docker image)

- updated mid-csp-umbrella helm chart to rely on mid-csp version 0.1.2 



0.6.10