Commit 4fce67c3 authored by Elisabetta Giani's avatar Elisabetta Giani
Browse files

CT-217: Acquire TangoMonitor before executing the fine check on the 

observing states during configuration.
Some tests show a dead-lock condition if the Abort command is received
just before the call to succeeded/failed.
Added a check on the GoToIdle command invoked on a sub-element,
Simplified the configure thread.
Enabled logging in pytest.
parent cff6ac80

csp-lmc-common/csp_lmc_common/CspSubarray.py

+36 −51
Original line number Diff line number Diff line
@@ -811,27 +811,16 @@ class CspSubarray(SKASubarray): 
            target_device._list_dev_completed_task[cmd_name] = []

            target_device._cmd_progress[cmd_name] = 0

            target_device._cmd_duration_measured[cmd_name] = 0

            stop_configuration = False

            # flag to signal when configuration ends on a sub-array sub-component

            device_done = defaultdict(lambda:False)

            # inside the end-less lop check the obsState of each sub-component

            device_list = input_arg[0]

            self.logger.info("Trhead started at {}".format(time.time()))

            while True:

                if target_device._abort_obs_event.is_set():

                    self.logger.info("Received and ABORT request during configuration {}".format(time.time()))

                command_progress = 0

                for device in device_list:

                    self.logger.info("Current device {} obsState is {}".format(device,

                                                                     ObsState(target_device._sc_subarray_obs_state[device]).name))

                    if target_device._abort_obs_event.is_set():

                        dev_successful_state = ObsState.ABORTED

                        cmd_name = 'abort'

                        if not stop_configuration:

                            command_start_time = time.time()

                            stop_configuration = True

                            device_done.clear()

                            target_device._reconfiguring = False

                    if device_done[device] == True:

                        continue

                    # if the sub-component execution flag is no more RUNNING, the command has
@@ -886,8 +875,13 @@ class CspSubarray(SKASubarray): 
                target_device._cmd_progress[cmd_name] = command_progress

                if any(target_device._sc_subarray_obs_state[device] == ObsState.CONFIGURING for device in device_list):

                    #target_device._reconfiguring = False

                    self.logger.info("device {} is in {}: reconfiguring is:{}".format(device, ObsState(target_device._sc_subarray_obs_state[device]).name,

                    self.logger.info("device {} is in {}: reconfiguring is:{}".format(device, 

                                                                                      ObsState(target_device._sc_subarray_obs_state[device]).name,

                                                                                      target_device._reconfiguring))

                # check if the abort flag is set

                if target_device._abort_obs_event.is_set():

                    self.logger.info("Received an ABORT request during configuration {}".format(time.time()))

                    return

                if all(value == True for value in device_done.values()):

                    self.logger.info("All devices have been handled at time {}!".format(time.time()))

                    break
@@ -897,14 +891,12 @@ class CspSubarray(SKASubarray): 
                    target_device._timeout_expired = True

                    self.logger.warning("Device went in timeout during configuration")

                    break

                # TODO:

                # check for abort request.

                time.sleep(0.1)

            # end of the while loop

            # acquire the mutex during the check of configuration success/failure. We don't want

            # to receive an boart during this phase otherwise could happen strange situation

            self.logger.info("GOING To lock mutex at {}".format(time.time()))

            with target_device._mutex_obs_state:

            # to receive an abort during this phase otherwise could happen strange situation

            self.logger.info("Going to lock TangoMonitor at {}".format(time.time()))

            with tango.AutoTangoMonitor(self.target):

                # check for timeout/failure conditions on each sub-component

                if any(target_device._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.TIMEOUT for device in device_list):

                    target_device._timeout_expired = True
@@ -918,27 +910,19 @@ class CspSubarray(SKASubarray): 
                self.logger.info("CspSubarray failure flag:{}".format(target_device._failure_raised))

                self.logger.info("CspSubarray timeout flag:{}".format(target_device._timeout_expired))

                if target_device._abort_obs_event.is_set():

                    if target_device._timeout_expired or target_device._failure_raised:

                        return self.failed()

                    self.logger.info("Abort configure ends with success!! {}".format(time.time()))

                    if all(target_device._sc_subarray_obs_state[fqdn] == ObsState.ABORTED for fqdn in device_list):

                        return target_device.abort_cmd_obj.succeeded()

                    return target_device.abort_cmd_obj.abort_monitoring(device_list)



                if target_device._timeout_expired or target_device._failure_raised:

                    # if failure/timeout found check if the CBF subarray is configured. In

                    # this case the CSP.LMC Subarray obsState is set to READY.

                    if target_device._sc_subarray_obs_state[target_device.CbfSubarray] == ObsState.READY:

                        return self.succeeded()

                    self.logger.info("Configure ends with failure")

                    return self.failed()

                    return

                try:

                    if all(target_device._sc_subarray_obs_state[fqdn] == ObsState.READY for fqdn in device_list):

                        target_device._valid_scan_configuration = input_arg[1]

                        target_device._cmd_duration_measured[cmd_name] = time.time() - command_start_time

                        target_device._cmd_progress[cmd_name] = 100

                        target_device._last_executed_command = cmd_name

                    self.logger.info("Configure ends with success!! {}".format(time.time()))

                        self.logger.info("1 Configure ends with success!! {}".format(time.time()))

                        return self.succeeded()

                except Exception as e:

                    self.logger.error(e)

		self.logger.info("Configure ends with failure!! {}".format(time.time()))

                return self_failed



        def validate_scan_configuration(self, argin):

            """
@@ -1043,8 +1027,6 @@ class CspSubarray(SKASubarray): 
            stop_scan = False

            # inside the end-less loop check the obsState of each sub-component

            while True:

                #self.logger.info("abort:{}".format(target_device._abort_obs_event.is_set()))

                #self.logger.info("end:{}".format(target_device._end_scan_event.is_set()))

                if target_device._stop_thread[cmd_name]:

                    target_device._stop_thread[cmd_name] = False

                    self.logger.info("STOPPING THE THREAD!!!")
@@ -1209,24 +1191,26 @@ class CspSubarray(SKASubarray): 
            if not any(device._sc_subarray_assigned_fqdn):

                # need to add a check also on PSTBeams belonging to subarray

                device_list = device._sc_subarray_fqdn

            self.logger.info("ABORT do at time {}".format(time.time()))

            try:

                abort_group = tango.Group("AbortGroup")

                for fqdn in device_list:

                    abort_group.add(fqdn)

                answer_id = abort_group.command_inout_asynch("Abort")

                device._abort_obs_event.set() 

                self.logger.info("abort is set? {}".format(device._abort_obs_event.is_set()))

                group_reply = abort_group.command_inout_reply(answer_id)

                '''

                if all ((reply.get_data())[0] == ResultCode.FAILED for reply in group_reply):

                    return (ResultCode.FAILED, "Abort FAILED")

                '''    

                device._abort_obs_event.set() 

                self.logger.info("abort is set? {}".format(device._abort_obs_event.is_set()))

            except (TypeError, Exception) as ex:

                self.logger.info(str(ex))

                self.logger.error("TANGO Group command failed")

                return (ResultCode.FAILED, "Abort command failed")

            message = "Abort command completed STARTED"

            self.logger.info(message)

            if all(device._command_thread[cmd_name].is_alive() == False for cmd_name in device._command_thread.keys()):

            #if all(device._command_thread[cmd_name].is_alive() == False for cmd_name in device._command_thread.keys()):

            device._command_thread['abort'] = threading.Thread(target=self.abort_monitoring,

                                                               name="Thread-Abort",

                                                               args=(device_list,))
@@ -1294,6 +1278,8 @@ class CspSubarray(SKASubarray): 
            # set all READY devices in IDLE

            for fqdn in device_list:

                proxy = device._sc_subarray_proxies[fqdn]

                if device._sc_subarray_obs_state[fqdn] == ObsState.EMPTY:

                    continue

                if device._sc_subarray_obs_state[fqdn] == ObsState.READY:

                    proxy.GoToIdle()

            for fqdn in device_list:
@@ -1372,12 +1358,12 @@ class CspSubarray(SKASubarray): 
    class GoToIdleCommand(SKASubarray.EndCommand):

        def do(self):

            target_device = self.target

            target_device._failure_raised = False

            device_list = target_device._sc_subarray_assigned_fqdn

            if not any(target_device._sc_subarray_assigned_fqdn):

                # need to add a check also on PSTBeams belonging to subarray

                device_list = target_device._sc_subarray_fqdn

            try:

                self.logger.info("Creating group for GoToIdle {}".format(device_list))

                sc_group = tango.Group("GoToIdleGroup")

                for device in device_list:

                    sc_group.add(device)
@@ -1388,12 +1374,13 @@ class CspSubarray(SKASubarray): 
            answers = sc_group.command_inout("GoToIdle")

            for reply in answers:

                if reply.has_failed():

                    target_device._failure_raised = True

                    for err in reply.get_err_stack():

                        self.logger.error("device {}: {}-{}".format(reply.dev_name(), err.desc, err.reason))

                else:

                    (result_code,msg) = reply.get_data()

                    self.logger.error("device {}: {}".format(reply.dev_name(), msg))

            if any(target_device._sc_subarray_obs_state[device]== ObsState.FAULT for device in  device_list):

            if any(target_device._sc_subarray_obs_state[device] == ObsState.FAULT for device in  device_list) or target_device._failure_raised:

                return (ResultCode.FAILED, "GoToIdle Command FAILED")

            return (ResultCode.OK, "GoToIdle command executed OK")
@@ -1586,7 +1573,6 @@ class CspSubarray(SKASubarray): 
                                                                                               evt.cmd_name))

                        if evt.cmd_name.lower() == 'configurescan':

                            self._reconfiguring = False

                            self.logger.info("Subarray is in reconfiguring...")

                    if evt.argout[0] == ResultCode.FAILED:

                        self.logger.info("Failure in Device {} while processing the command {}".format(evt.device.dev_name(),

                                                                                               evt.cmd_name))
@@ -2876,7 +2862,6 @@ class CspSubarray(SKASubarray): 
        :return:'DevVarLongStringArray'

        """

        self.logger.info("CALL ABORT at time {}".format(time.time()))

        with self._mutex_obs_state:

        handler = self.get_command_object("Abort")

        (result_code, message) = handler()

        return [[result_code], [message]]

csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py

+7 −8
Original line number Diff line number Diff line
@@ -628,11 +628,11 @@ class MidCspSubarrayBase(CspSubarray): 
        self._last_executed_command = cmd_name

        # update the progress counter at the end of the loop

        self._cmd_progress[cmd_name] = self._sc_subarray_cmd_progress[device][cmd_name]

        self.logger.info("CspSubarray failure flag:{}".format(self._failure_raised))

        self.logger.info("CspSubarray timeout flag:{}".format(self._timeout_expired))

        # check for error conditions

        if self._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.FAILED:

            self._failure_raised = True

        if self._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.TIMEOUT:

            self._timeout_expired = True

        if self._sc_subarray_state[self.CbfSubarray] not in [tango.DevState.OFF,

                                                             tango.DevState.ON]:

            self._failure_message[cmd_name] += ("Device {} is in {} State".format(device,
@@ -640,14 +640,13 @@ class MidCspSubarrayBase(CspSubarray): 
            self.logger.warning(self._failure_message[device])

            self._failure_raised = True

            self.logger.error("ReleaseReceptors ended with failure")

            return self._releaseresources_cmd_obj.failed()

        if self._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.TIMEOUT:

            self._timeout_expired = True

            self.logger.error("ReleaseReceptors ended with timeout")

            return self._releaseresources_cmd_obj.failed()

        # reset the command exeuction state

        self.logger.info("CspSubarray failure flag:{}".format(self._failure_raised))

        self.logger.info("CspSubarray timeout flag:{}".format(self._timeout_expired))

        self._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.IDLE

        self._cmd_execution_state[cmd_name] = CmdExecState.IDLE

        if self._failure_raised  or self._timeout_expired:

            return self._releaseresources_cmd_obj.failed()

        # reset the command exeuction state

        self.logger.info("Receptors removed with success")

        return self._releaseresources_cmd_obj.succeeded()

csp-lmc-mid/csp_lmc_mid/receptors.py

+0 −1
Original line number Diff line number Diff line
@@ -117,7 +117,6 @@ class Receptors: 
        assigned_receptors = []

        sub_id = int(_sub_id)

        try:

            self.logger.info(f"CbfSubarray: {self.device.CbfSubarray}")

            proxy = self.connect(self.device.CbfSubarray)

            #receptor_membership = self.subarray_affiliation()

            #assigned_receptors = [receptor_id + 1 for receptor_id, e in enumerate(receptor_membership) if e == int(sub_id)]

csp-lmc-mid/setup.cfg

+1 −0
Original line number Diff line number Diff line
@@ -14,6 +14,7 @@ source = csp-lmc-mid 


[tool:pytest]

testpaths = tests

log_cli = True

addopts = --forked

          --verbose

	  --cov=csp_lmc_mid

csp-lmc-mid/tests/integration/MidCspSubarray_test.py

+6 −6
Original line number Diff line number Diff line
@@ -98,7 +98,6 @@ class TestCspSubarray(TestBase): 
        """

        Set the subarray state to OFF-EMPTY

        """

        self._setup_subarray()

        self.midcsp_subarray01.Off()

        prober_subarray_state = Probe(self.midcsp_subarray01, "State", DevState.OFF, f"CSP Master not ON")

        Poller(4, 0.2).check(prober_subarray_state)
@@ -109,6 +108,7 @@ class TestCspSubarray(TestBase): 
        The final subarray state is ON-IDLE

        """

        receptor_list = self.midcsp_master.unassignedReceptorIDs

        LOGGER.info(f"receptor: {receptor_list}")

        # assert the array is not empty

        assert receptor_list.any()

        # assign all available receptors to the subarray
@@ -139,6 +139,7 @@ class TestCspSubarray(TestBase): 
        assert obs_state == ObsState.IDLE

        receptors = self.midcsp_subarray01.assignedReceptors

        LOGGER.info("Receptor assigned to the subarray {}".format(receptors))

        assert not receptors.any()

        try:

            LOGGER.info("invoke remove all receptors")

            self.midcsp_subarray01.ReleaseAllResources()
@@ -149,7 +150,8 @@ class TestCspSubarray(TestBase): 
        Poller(4, 0.2).check(prober_obs_state)

        obs_state = self.midcsp_subarray01.obsState

        LOGGER.info("obs_state:{}".format(obs_state))

        LOGGER.info("EMPTY:{}".format(ObsState.EMPTY))

        receptors = self.midcsp_subarray01.assignedReceptors

        LOGGER.info("Receptor assigned to the subarray {}".format(receptors))



    def _goto_idle(self):

        """
@@ -333,7 +335,6 @@ class TestCspSubarray(TestBase): 
        prober_subarray_obsstate = Probe(self.midcsp_subarray01, "obsState", ObsState.EMPTY, f"CSP Subarray not EMPTY")

        Poller(4, 0.2).check(prober_subarray_obsstate)

        receptors = self.midcsp_subarray01.assignedReceptors

        LOGGER.info(f'type of {type(receptors)}')

        LOGGER.info(f'list of receptors{receptors}')

        assert not receptors.any(), f"CSP Subarray is not empty"
@@ -775,9 +776,8 @@ class TestCspSubarray(TestBase): 
        Poller(4, 0.2).check(prober_obs_state)



    @pytest.mark.abort

    #@pytest.mark.parametrize("elapsed_time", [0.4, 0.41, 0.42, 0.43, 0.44])

    @pytest.mark.parametrize("elapsed_time", [0.42, 0.43, 0.44])

    @pytest.mark.parametrize('execution_number', range(2))

    @pytest.mark.parametrize("elapsed_time", [0.4, 0.41, 0.415, 0.42, 0.43, 0.44, 0.45])

    @pytest.mark.parametrize('execution_number', range(1))

    def test_send_abort_WHILE_in_configuring(self, elapsed_time, execution_number):

        """

        Test that the subarray is able to handle the Abort command when issued