CT-67: fix some issues in Abort handling during configuration. (04ad8691) · Commits · Gianluca Marotta / csp-lmc-old

csp-lmc-common/HISTORY

+6 −0

Original line number	Diff line number	Diff line
		0.6.5
		- Implemented Abort command.
		- Use mutex to avoid Abort invkoking while the configuring thread is
		checking the global CSP observing state.


		0.6.4
		- CSP Subarrays are enabled when the On command is invoked on the CspMaster
		- fix EndScan: command invoked synchronously

csp-lmc-common/csp_lmc_common/CspSubarray.py

+59 −75

Original line number	Diff line number	Diff line
		@@ -315,8 +315,6 @@ class CspSubarray(SKASubarray):
		# change the global TANGO model to PUSH_CALLBACK.
		apiutil = tango.ApiUtil.instance()
		apiutil.set_asynch_cb_sub_model(tango.cb_sub_model.PUSH_CALLBACK)

		message = "CspSubarray Init command completed OK"
		self.logger.info(message)
		return (result_code, message)

		@@ -489,7 +487,7 @@ class CspSubarray(SKASubarray):
		self.logger.debug("_config_delay_expected :{}".format(target_device._config_delay_expected))
		# invoke the constructor for the command thread
		thread_args = [target_device._sc_subarray_assigned_fqdn, argin]
		target_device._command_thread['configurescan'] = threading.Thread(target=self._configure_scan,
		target_device._command_thread['configurescan'] = threading.Thread(target=self.configure_scan,
		name="Thread-Configure",
		args=(thread_args,))
		target_device._abort_obs_event.clear()
		@@ -500,7 +498,7 @@ class CspSubarray(SKASubarray):
		self.logger.info(message)
		return (ResultCode.STARTED, message)

		def _configure_scan(self, input_arg, **args_dict):
		def configure_scan(self, input_arg, **args_dict):
		"""
		Thread target function invoked from Configure method.
		It monitors the obsState value of each sub-array sub-component
		@@ -517,13 +515,10 @@ class CspSubarray(SKASubarray):
		:return: None
		"""
		target_device = self.target
		self.logger.info("__configure scan started!!")
		cmd_name = 'configurescan'
		dev_successful_state = ObsState.READY
		# tango_cmd_name: is the TANGO command name with the capital letter
		# In the dictionary keys, is generally used the command name in lower letters
		self.logger.info("cmd_name: {} dev_state: {}".format(cmd_name,
		dev_successful_state))
		command_start_time = time.time()
		target_device._num_dev_completed_task[cmd_name] = 0
		target_device._list_dev_completed_task[cmd_name] = []
		@@ -534,7 +529,6 @@ class CspSubarray(SKASubarray):
		device_done = defaultdict(lambda:False)
		# inside the end-less lop check the obsState of each sub-component
		device_list = input_arg[0]
		self.logger.info("device_list:{}".format(device_list))
		while True:
		if target_device._abort_obs_event.is_set():
		self.logger.info("Received and ABORT request during configuration")
		@@ -558,7 +552,6 @@ class CspSubarray(SKASubarray):
		self.logger.info("current device {} state:{}".format(device,
		ObsState(target_device._sc_subarray_obs_state[device]).name))
		if target_device._sc_subarray_obs_state[device] == dev_successful_state:
		self.logger.info("Reconfiguring is:{}".format(target_device._reconfiguring))
		if not target_device._reconfiguring:
		self.logger.info("Command {} ended with success on device {}.".format(cmd_name,
		device))
		@@ -588,7 +581,6 @@ class CspSubarray(SKASubarray):
		self.logger.warning(msg)
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.TIMEOUT
		device_done[device] = True
		self.logger.info("elapsed_time:{} device {}".format(elapsed_time, device))
		# check if sub-element command ended throwing an exception: in this case the
		# 'cmd_ended_cb' callback is invoked.
		if target_device._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.FAILED or \
		@@ -603,8 +595,8 @@ class CspSubarray(SKASubarray):
		target_device._sc_subarray_cmd_exec_state[device][cmd_name]))
		target_device._cmd_progress[cmd_name] = command_progress
		if any(target_device._sc_subarray_obs_state[device] == ObsState.CONFIGURING for device in device_list):
		self.logger.info("reconfiguring is:{}".format(target_device._reconfiguring))
		target_device._reconfiguring = False
		self.logger.info("Reconfiguring is:{}".format(target_device._reconfiguring))
		if all(value == True for value in device_done.values()):
		self.logger.info("All devices have been handled!")
		break
		@@ -616,33 +608,36 @@ class CspSubarray(SKASubarray):
		break
		# TODO:
		# check for abort request.
		self.logger.info("Sleep for 0.1")
		time.sleep(0.1)
		# end of the while loop
		# acquire the mutex during the check of configuration success/failure. We don't want
		# to receive an boart during this phase otherwise could happen strange situation
		with target_device._mutex_obs_state:
		# check for timeout/failure conditions on each sub-component
		if any(target_device._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.TIMEOUT for device in device_list):
		target_device._timeout_expired = True
		if any(target_device._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.FAILED for device in device_list):
		target_device._failure_raised = True
		# reset sub-component execution flag
		# reset the sub-element command execution flag
		for device in device_list:
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.IDLE

		self.logger.info("CspSubarray failure flag:{}".format(target_device._failure_raised))
		self.logger.info("CspSubarray timeout flag:{}".format(target_device._timeout_expired))
		# acquire the mutex during the check of configuration success/failure. We don't want
		# to receive an boart during this phase otherwise could happen strange situation
		if target_device._abort_obs_event.is_set():
		if target_device._timeout_expired or target_device._failure_raised:
		return target_device.abort_cmd_obj.failed()
		self.logger.info("Abort configure ends with success!!")
		if all(target_device._sc_subarray_obs_state[fqdn] == ObsState.ABORTED for fqdn in device_list):
		return target_device.abort_cmd_obj.succeeded()
		return target_device.abort_cmd_obj.abort_monitoring(device_list)

		with target_device._mutex_obs_state:
		if target_device._timeout_expired or target_device._failure_raised:
		# if failure/timeout found check if the CBF subarray is configured. In
		# this case the CSP.LMC Subarray obsState is set to READY.
		if target_device._sc_subarray_obs_state[target_device.CbfSubarray] == ObsState.READY:
		return target_device.configure_cmd_obj.succeeded()
		self.logger.info("Configure ends with failure")
		return target_device.configure_cmd_obj.failed()
		if all(target_device._sc_subarray_obs_state[fqdn] == ObsState.READY for fqdn in device_list):
		target_device._valid_scan_configuration = input_arg[1]
		@@ -719,14 +714,14 @@ class CspSubarray(SKASubarray):
		# TODO: add check on the failed device. If CBF
		# throw an exception.
		# invoke the constructor for the command thread
		target_device._command_thread['scan'] = threading.Thread(target=self.__monitor_scan_execution,
		target_device._command_thread['scan'] = threading.Thread(target=self.monitor_scan_execution,
		name="Thread-Scan",
		args=(target_device._sc_subarray_assigned_fqdn,))
		target_device._cmd_execution_state['scan'] = CmdExecState.RUNNING
		target_device._command_thread['scan'].start()
		return (ResultCode.STARTED, "Scan command started")

		def __monitor_scan_execution(self, device_list):
		def monitor_scan_execution(self, device_list):
		cmd_name = 'scan'
		target_device = self.target
		dev_successful_state = ObsState.READY
		@@ -764,21 +759,18 @@ class CspSubarray(SKASubarray):
		if target_device._sc_subarray_obs_state[device] == ObsState.FAULT:
		target_device._failure_raised = True
		device_done[device] = True
		self.logger.info("abort timeout value: {}".format(target_device._sc_subarray_cmd_duration_expected[device][cmd_name]))
		if elapsed_time > target_device._sc_subarray_cmd_duration_expected[device][cmd_name]:
		target_device._cmd_execution_state[cmd_name] = CmdExecState.IDLE
		target_device.timeout_expired = True
		device_done[device] = True
		if any(device_done.values()) and all(value == True for value in device_done.values()):
		break
		self.logger.info("Going to sleep")
		time.sleep(0.1)

		target_device._cmd_execution_state[cmd_name] = CmdExecState.IDLE
		target_device._cmd_progress[cmd_name] = 100
		elapsed_time = time.time() - starting_time
		self.logger.info("Scan elapsed time:{}".format(elapsed_time))
		self.logger.info("abort:{}".format(target_device._abort_obs_event.is_set()))
		if target_device._end_scan_event.is_set():
		return
		if target_device._abort_obs_event.is_set():
		@@ -803,7 +795,6 @@ class CspSubarray(SKASubarray):
		except Exception:
		self.logger.error("TANGO Group command failed")
		return (ResultCode.FAILED, "EndScan Command FAILED")
		self.logger.info("Issue EndScan")
		answers = sc_group.command_inout("EndScan")
		target_device._end_scan_event.set()
		for reply in answers:
		@@ -839,8 +830,6 @@ class CspSubarray(SKASubarray):
		continue
		if target_device._sc_subarray_obs_state[device] == ObsState.READY:
		(result_code, msg) = target_device.gotoidle_cmd_obj.do()
		self.logger.info("result_code: {}".format(result_code))
		#return (ResultCode.OK, "ObsReset command OK")
		return (result_code, msg)
		for device in devices_to_reset:
		try:
		@@ -854,29 +843,26 @@ class CspSubarray(SKASubarray):
		self.logger.warning(tango_err.args[0].desc)
		# TODO: address this case!
		# signal the failure raising the failure flag?
		target_device._command_thread['obsreset'] = threading.Thread(target=self._monitor_obsreset,
		target_device._command_thread['obsreset'] = threading.Thread(target=self.monitor_obsreset,
		name="Thread-ObsReset",
		args=(devices_to_reset,))
		target_device._command_thread['obsreset'].start()
		return (ResultCode.STARTED, "ObsReset command executed STARTED")

		def _monitor_obsreset(self, device_list):
		def monitor_obsreset(self, device_list):
		cmd_name = 'obsreset'
		target_device = self.target
		dev_successful_state = ObsState.IDLE
		target_device._cmd_progress[cmd_name] = 0
		device_done = defaultdict(lambda:False)
		# inside the end-less loop check the obsState of each sub-component
		self.logger.debug("device_list:{}".format(device_list))
		while True:
		self.logger.info("Going to sleep")
		time.sleep(0.1)
		for device in device_list:
		if device_done[device] == True:
		continue
		# if the sub-component execution flag is no more RUNNING, the command has
		# ended with or without success. Go to check next device state.
		self.logger.info("device {} obs_state:{}".format(device, target_device._sc_subarray_obs_state[device]))
		if target_device._sc_subarray_obs_state[device] == dev_successful_state:
		self.logger.info("Command {} ended with success on device {}.".format(cmd_name,
		device))
		@@ -920,30 +906,27 @@ class CspSubarray(SKASubarray):
		abort_group = tango.Group("AbortGroup")
		for fqdn in device_list:
		abort_group.add(fqdn)
		answer_id = abort_group.command_inout_asynch("Abort")
		group_reply = abort_group.command_inout_reply(answer_id)
		'''
		proxy = device._sc_subarray_proxies[fqdn]
		self.logger.info("Sending Abort command")
		(result_code, msg) = proxy.Abort()
		if result_code in [ResultCode.OK, ResultCode.STARTED]:
		device._abort_obs_event.set()
		self.logger.info("After Abort")
		if all ((reply.get_data())[0] == ResultCode.FAILED for reply in group_reply):
		return (ResultCode.FAILED, "Abort FAILED")
		'''
		with device._mutex_obs_state:
		answers = abort_group.command_inout_asynch("Abort")
		device._abort_obs_event.set()
		self.logger.info("abort is set? {}".format(device._abort_obs_event.is_set()))
		except (TypeError, Exception) :
		except (TypeError, Exception) as ex:
		self.logger.info(str(ex))
		self.logger.error("TANGO Group command failed")
		message = "Abort command completed STARTED"
		self.logger.info(message)
		if all(device._command_thread[cmd_name].is_alive() == False for cmd_name in device._command_thread.keys()):
		device._command_thread['abort'] = threading.Thread(target=self._abort_monitoring,
		device._command_thread['abort'] = threading.Thread(target=self.abort_monitoring,
		name="Thread-Abort",
		args=(device_list,))
		device._command_thread['abort'].start()
		return (ResultCode.STARTED, message)

		def _abort_monitoring(self,device_list):
		def abort_monitoring(self, device_list):
		cmd_name = 'abort'
		target_device = self.target
		device_done = defaultdict(lambda:False)
		@@ -1669,7 +1652,6 @@ class CspSubarray(SKASubarray):
		# Class private methods
		# ----------------


		def _init_state_model(self):
		"""
		Sets up the state model for the device
		@@ -2661,6 +2643,25 @@ class CspSubarray(SKASubarray):
		return [[result_code], [message]]
		# PROTECTED REGION END # // CspSubarray.GoToIdle

		@command(
		dtype_out='DevVarLongStringArray',
		doc_out="(ReturnType, 'informational message')",
		)
		@DebugIt()
		def Abort(self):
		# PROTECTED REGION ID(CspSubarray.Abort) ENABLED START #
		"""
		Change obsState to ABORTED.

		:return:'DevVarLongStringArray'
		"""
		with self._mutex_obs_state:
		handler = self.get_command_object("Abort")
		(result_code, message) = handler()
		return [[result_code], [message]]

		# PROTECTED REGION END # // CspSubarray.Abort

		'''
		@command(
		dtype_out='DevVarLongStringArray',
		@@ -2837,23 +2838,6 @@ class CspSubarray(SKASubarray):
		self._cmd_execution_state['removetimingbeams'] = CmdExecState.RUNNING
		# PROTECTED REGION END # // CspSubarray.RemoveTimingBeams

		@command(
		dtype_out='DevVarLongStringArray',
		doc_out="(ReturnType, 'informational message')",
		)
		@DebugIt()
		def Abort(self):
		# PROTECTED REGION ID(CspSubarray.Abort) ENABLED START #
		"""
		Change obsState to ABORTED.

		:return:'DevVarLongStringArray'
		"""
		handler = self.get_command_object("Abort")
		(result_code, message) = handler()
		return [[result_code], [message]]

		# PROTECTED REGION END # // CspSubarray.Abort
		'''

		# ----------

csp-lmc-common/csp_lmc_common/release.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -10,7 +10,7 @@
		"""Release information for Python Package"""

		name = """csp-lmc-common"""
		version = "0.6.4"
		version = "0.6.5"
		version_info = version.split(".")
		description = """SKA CSP.LMC Common Software"""
		author = "INAF-OAA"

csp-lmc-common/docker/.release

+3 −3

Original line number	Diff line number	Diff line
		@@ -9,7 +9,7 @@
		"""Release information for Python Package"""

		name = """csplmc-common"""
		version = "0.6.4"
		version = "0.6.5"
		version_info = version.split(".")
		description = """SKA CSP.LMC Common Classe"""
		author = "E.G"
		@@ -18,5 +18,5 @@ license = """BSD-3-Clause"""
		url = """www.tango-controls.org"""
		copyright = """"""

		release=0.6.4
		tag=csp-lmc-common-0.6.4
		release=0.6.5
		tag=csp-lmc-common-0.6.5

csp-lmc-mid/HISTORY

+4 −0

Original line number	Diff line number	Diff line
		0.6.5:
		- use csp-lmc-common version 0.6.5
		- added test for Abort.

		0.6.4:
		- use csp-lmc-common version 0.6.4
		- Receptor class uses the assignedReceptors attribute (exported by the MidCspSubarray)