CT-67: Added handlingof Abort during configuraiton. (5a046b19) · Commits · Gianluca Marotta / csp-lmc-old

csp-lmc-common/csp_lmc_common/CspSubarray.py

+118 −47

Original line number	Diff line number	Diff line
		@@ -153,6 +153,7 @@ class CspSubarray(SKASubarray):
		device._sc_subarray_admin_mode = defaultdict(lambda: AdminMode.NOT_FITTED)
		device._sc_subarray_obs_state = defaultdict(lambda: ObsState.IDLE)
		device._sc_subarray_obs_mode = defaultdict(lambda: ObsMode.IDLE)
		device._mutex_obs_state = threading.Lock()
		device._csp_tango_db = tango.Database()
		# read the CSP memorized attributes from the TANGO DB.
		# Note: a memorized attribute has defined the attribute
		@@ -228,7 +229,7 @@ class CspSubarray(SKASubarray):
		# Implemented as a nested default dictionary
		# keys: FQDN
		# values: default dictionary (keys: command name, values: the duration (in sec))
		device._sc_subarray_cmd_duration_expected = defaultdict(lambda: defaultdict(lambda: 10))
		device._sc_subarray_cmd_duration_expected = defaultdict(lambda: defaultdict(lambda: 5))

		# _sc_subarray_scan_configuration: report the scan configuration
		# for each sub-array sub-component (CBF, PSS subarray, PSTBeams)
		@@ -257,7 +258,7 @@ class CspSubarray(SKASubarray):
		# Implemented asdefault dictionary
		# keys: command name ('addsearchbeams','configurescan'....)
		# values: the duration (in sec)
		device._cmd_duration_expected = defaultdict(lambda: 0)
		device._cmd_duration_expected = defaultdict(lambda: 5)

		# _cmd_duration_measured: report the measured duration (in sec.) for
		# a long-running command
		@@ -528,20 +529,34 @@ class CspSubarray(SKASubarray):
		target_device._list_dev_completed_task[cmd_name] = []
		target_device._cmd_progress[cmd_name] = 0
		target_device._cmd_duration_measured[cmd_name] = 0
		stop_configuration = False
		# flag to signal when configuration ends on a sub-array sub-component
		device_done = defaultdict(lambda:False)
		# inside the end-less lop check the obsState of each sub-component
		device_list = input_arg[0]
		self.logger.info("device_list:{}".format(device_list))
		while True:
		if target_device._abort_obs_event.is_set():
		self.logger.info("Received and ABORT request during configuration")
		command_progress = 0
		for device in device_list:
		self.logger.info("Command {} obs_state: {}".format(cmd_name,
		target_device._sc_subarray_obs_state[device]))
		self.logger.info("Current device {} obsState is {}".format(device,
		ObsState(target_device._sc_subarray_obs_state[device]).name))
		if target_device._abort_obs_event.is_set():
		dev_successful_state = ObsState.ABORTED
		cmd_name = 'abort'
		if not stop_configuration:
		command_start_time = time.time()
		stop_configuration = True
		device_done.clear()
		target_device._reconfiguring = False
		if device_done[device] == True:
		continue
		# if the sub-component execution flag is no more RUNNING, the command has
		# ended with or without success. Go to check next device state.
		self.logger.info("Wait for final obsState = {}".format(ObsState(dev_successful_state).name))
		self.logger.info("current device {} state:{}".format(device,
		ObsState(target_device._sc_subarray_obs_state[device]).name))
		if target_device._sc_subarray_obs_state[device] == dev_successful_state:
		self.logger.info("Reconfiguring is:{}".format(target_device._reconfiguring))
		if not target_device._reconfiguring:
		@@ -567,8 +582,6 @@ class CspSubarray(SKASubarray):
		# Note: the second check, can be useful if the timeout event is not received
		# (for example for a temporary connection timeout)
		elapsed_time = time.time() - target_device._sc_subarray_cmd_starting_time[device]
		self.logger.debug("elapsed_time:{}".format(elapsed_time))

		if (elapsed_time > target_device._sc_subarray_cmd_duration_expected[device][cmd_name] or
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.TIMEOUT):
		msg = ("Timeout executing {} command on device {}".format(cmd_name, device))
		@@ -603,7 +616,7 @@ class CspSubarray(SKASubarray):
		break
		# TODO:
		# check for abort request.
		self.logger.debug("Sleep for 0.1")
		self.logger.info("Sleep for 0.1")
		time.sleep(0.1)
		# end of the while loop
		# check for timeout/failure conditions on each sub-component
		@@ -616,26 +629,29 @@ class CspSubarray(SKASubarray):

		self.logger.info("CspSubarray failure flag:{}".format(target_device._failure_raised))
		self.logger.info("CspSubarray timeout flag:{}".format(target_device._timeout_expired))
		# acquire the mutex during the check of configuration success/failure. We don't want
		# to receive an boart during this phase otherwise could happen strange situation
		if target_device._abort_obs_event.is_set():
		if target_device._timeout_expired or target_device._failure_raised:
		return target_device.abort_cmd_obj.failed()
		self.logger.info("Abort configure ends with success!!")
		return target_device.abort_cmd_obj.succeeded()

		with target_device._mutex_obs_state:
		if target_device._timeout_expired or target_device._failure_raised:
		# if failure/timeout found check if the CBF subarray is configured. In
		# this case the CSP.LMC Subarray obsState is set to READY.
		if target_device._sc_subarray_obs_state[target_device.CbfSubarray] == ObsState.READY:
		return target_device.configure_cmd_obj.succeeded()
		else :
		return target_device.configure_cmd_obj.failed()
		#TODO:
		# self.set_state(tango.DevState.FAULT)
		#

		if all(target_device._sc_subarray_obs_state[fqdn] == dev_successful_state for fqdn in device_list):
		if all(target_device._sc_subarray_obs_state[fqdn] == ObsState.READY for fqdn in device_list):
		target_device._valid_scan_configuration = input_arg[1]
		target_device._cmd_duration_measured[cmd_name] = time.time() - command_start_time
		target_device._cmd_progress[cmd_name] = 100
		target_device._last_executed_command = cmd_name
		self.logger.info("Configure ends with success!!")
		return target_device.configure_cmd_obj.succeeded()
		else:
		return target_device.configure_cmd_obj.failed()



		def validate_scan_configuration(self, argin):
		@@ -768,7 +784,7 @@ class CspSubarray(SKASubarray):
		if target_device._abort_obs_event.is_set():
		if target_device._failure_raised or target_device._timeout_expired:
		return target_device.abort_cmd_obj.failed()
		self.logger.info("Aborting ends with success!")
		self.logger.info("Abort of scan command ends with success!")
		return target_device.abort_cmd_obj.succeeded()

		class EndScanCommand(SKASubarray.EndScanCommand):
		@@ -900,22 +916,81 @@ class CspSubarray(SKASubarray):
		if not any(device._sc_subarray_assigned_fqdn):
		# need to add a check also on PSTBeams belonging to subarray
		device_list = device._sc_subarray_fqdn
		for fqdn in device_list:
		try:
		abort_group = tango.Group("AbortGroup")
		for fqdn in device_list:
		abort_group.add(fqdn)
		'''
		proxy = device._sc_subarray_proxies[fqdn]
		proxy.command_inout_asynch("Abort", device._cmd_ended_cb)
		except KeyError as key_err:
		self.logger.warning("No key {} found".format(key_err))
		device._sc_subarray_cmd_exec_state[fqdn]['abort'] = CmdExecState.FAILED
		except tango.DevFailed as tango_err:
		device._sc_subarray_cmd_exec_state[fqdn]['abort'] = CmdExecState.FAILED
		self.logger.warning(tango_err.args[0].desc)
		self.logger.info("Sending Abort command")
		(result_code, msg) = proxy.Abort()
		if result_code in [ResultCode.OK, ResultCode.STARTED]:
		device._abort_obs_event.set()
		self.logger.info("After Abort")
		'''
		with device._mutex_obs_state:
		answers = abort_group.command_inout_asynch("Abort")
		device._abort_obs_event.set()
		self.logger.info("abort is set? {}".format(device._abort_obs_event.is_set()))
		except (TypeError, Exception) :
		self.logger.error("TANGO Group command failed")
		message = "Abort command completed STARTED"
		self.logger.info(message)
		if all(device._command_thread[cmd_name].is_alive() == False for cmd_name in device._command_thread.keys()):
		device._command_thread['abort'] = threading.Thread(target=self._abort_monitoring,
		name="Thread-Abort",
		args=(device_list,))
		device._command_thread['abort'].start()
		return (ResultCode.STARTED, message)

		def _abort_monitoring(self,device_list):
		cmd_name = 'abort'
		target_device = self.target
		device_done = defaultdict(lambda:False)
		elapsed_time = 0
		starting_time = time.time()
		while True:
		for device in device_list:
		if device_done[device] == True:
		continue
		self.logger.info("device {} obs_state:{}".format(device, target_device._sc_subarray_obs_state[device]))
		if target_device._sc_subarray_obs_state[device] == ObsState.ABORTED:
		self.logger.info("Command {} ended with success on device {}.".format(cmd_name,
		device))
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.IDLE
		target_device._sc_subarray_cmd_progress[device][cmd_name] = 100
		# command success: step to next device
		device_done[device] = True
		# check if sub-element command ended throwing an exception: in this case the
		# 'cmd_ended_cb' callback is invoked.
		if target_device._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.FAILED or\
		target_device._sc_subarray_obs_state[device] == ObsState.FAULT:
		# execution ended for this sub-element, skip to the next one
		target_device._failure_raised = True
		device_done[device] = True
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.IDLE
		elapsed_time = time.time() - starting_time
		if elapsed_time > target_device._sc_subarray_cmd_duration_expected[device][cmd_name]:
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.TIMEOUT
		target_device._timeout_expired = True
		device_done[device] = True

		if any(device_done.values()) and all(value == True for value in device_done.values()):
		self.logger.info("All devices have been handled!")
		break
		self.logger.info("Going to sleep")
		time.sleep(0.1)

		# end of the while loop
		# check for timeout/failure conditions on each sub-component
		if target_device._failure_raised or target_device._timeout_expired:
		return target_device.abort_cmd_obj.failed()

		if all(target_device._sc_subarray_obs_state[fqdn] == ObsState.ABORTED for fqdn in device_list):
		target_device._cmd_progress[cmd_name] = 100
		target_device._last_executed_command = cmd_name
		self.logger.info("Abort ends with success")
		return target_device.abort_cmd_obj.succeeded()
		'''
		class GoToIdleCommand(ActionCommand):
		"""
		@@ -1175,19 +1250,14 @@ class CspSubarray(SKASubarray):
		try:
		if dev_name in self._sc_subarray_fqdn:
		if evt.attr_value.name.lower() == "state":
		self.logger.info("device: {} state value {}".format(dev_name,evt.attr_value.value))
		self._sc_subarray_state[dev_name] = evt.attr_value.value
		elif evt.attr_value.name.lower() == "healthstate":
		self.logger.info("device: {} healthstate value {}".format(dev_name,evt.attr_value.value))
		self._sc_subarray_health_state[dev_name] = evt.attr_value.value
		elif evt.attr_value.name.lower() == "adminmode":
		self.logger.info("device: {} adminMode value {}".format(dev_name,evt.attr_value.value))
		self._sc_subarray_admin_mode[dev_name] = evt.attr_value.value
		elif evt.attr_value.name.lower() == "obsstate":
		self.logger.info("device: {} obsState value {}".format(dev_name,evt.attr_value.value ))
		self._sc_subarray_obs_state[dev_name] = evt.attr_value.value
		elif evt.attr_value.name.lower() == "obsmode":
		self.logger.debug("device: {} obsMode value {}".format(dev_name, evt.attr_value.value ))
		self._sc_subarray_obs_mode[dev_name] = evt.attr_value.value
		else:
		log_msg = ("Attribute {} not still "
		@@ -1199,7 +1269,8 @@ class CspSubarray(SKASubarray):
		self.logger.warning(log_msg)
		return

		log_msg = "New value for {} is {}".format(str(evt.attr_name),
		log_msg = "Received event on {}/{}: {}".format(dev_name,
		str(evt.attr_value.name),
		str(evt.attr_value.value))
		self.logger.info(log_msg)
		# update CSP sub-array SCM

csp-lmc-mid/tests/MidCspSubarray_test.py

+49 −1

Original line number	Diff line number	Diff line
		@@ -83,6 +83,10 @@ class TestCspSubarray(TestBase):
		return
		if obs_state == ObsState.READY:
		self._goto_idle()
		if obs_state == ObsState.ABORTED:
		self.midcsp_subarray01.ObsReset()
		prober_obs_state = Probe(self.midcsp_subarray01, "obsState", ObsState.IDLE, f"CSP Subarray not IDLE")
		Poller(4, 0.2).check(prober_obs_state)
		obs_state = self.midcsp_subarray01.obsState
		if obs_state == ObsState.IDLE:
		self._release_all_receptors()
		@@ -419,11 +423,31 @@ class TestCspSubarray(TestBase):
		obs_state = self.midcsp_subarray01.obsState
		LOGGER.info("Issue the EndScan command")
		self.midcsp_subarray01.EndScan()
		time.sleep(1)
		#time.sleep(1)
		prober_obs_state = Probe(self.midcsp_subarray01, 'obsState', ObsState.READY,
		f"Wrong CSP Subarray obsState")
		Poller(10, 0.2).check(prober_obs_state)

		def test_abort_scan(self):
		"""
		Test that a subarray is able to process the
		Scan command when its ObsState is READY
		"""
		self._configure_scan()
		state = self.midcsp_subarray01.State()
		obs_state = self.midcsp_subarray01.obsState
		LOGGER.info("CSP Subarray State before exercise :{}-{}".format(state, ObsState(obs_state).name))
		LOGGER.info("Issue the Scan command")
		self.midcsp_subarray01.Scan("11")
		prober_obs_state = Probe(self.midcsp_subarray01, 'obsState', ObsState.SCANNING,
		f"Wrong CSP Subarray obsState")
		Poller(10, 0.2).check(prober_obs_state)
		obs_state = self.midcsp_subarray01.obsState
		LOGGER.info("Issue the Abort command")
		self.midcsp_subarray01.Abort()
		prober_obs_state = Probe(self.midcsp_subarray01, 'obsState', ObsState.ABORTED,
		f"Wrong CSP Subarray obsState")
		Poller(10, 0.2).check(prober_obs_state)

		def test_obsreset_cbf_AFTER_invalid_configuration(self):
		"""
		@@ -452,6 +476,30 @@ class TestCspSubarray(TestBase):
		prober_obs_state = Probe(self.midcsp_subarray01, "obsState", ObsState.IDLE, f"CSP Subarray not IDLE")
		Poller(4, 0.2).check(prober_obs_state)

		def test_send_abort_WHILE_in_configuring(self):
		"""
		Configure the CSP Subarray with a JSon string including
		the new ADR4 fields.
		"""
		self._setup_subarray()
		self._assign_receptors()
		state = self.midcsp_subarray01.State()
		obs_state = self.midcsp_subarray01.obsState
		LOGGER.info("CSP Subarray State before exercise :{}-{}".format(state, ObsState(obs_state).name))
		# exercise the device
		LOGGER.info(f"Configuring CSP subarray01")
		f = open(file_path + "/acceptance_tests/test_ConfigureScan_ADR4.json")
		(result_code, msg) = self.midcsp_subarray01.Configure(f.read().replace("\n", ""))
		f.close()
		time.sleep(0.1)
		self.midcsp_subarray01.Abort()
		# check
		prober_subarray_obstate = Probe(self.midcsp_subarray01, 'obsState', ObsState.ABORTED,
		f"Wrong CSP Subarray obsState {self.midcsp_subarray01.obsState}")
		Poller(5, 0.2).check(prober_subarray_obstate)
		obs_state = self.midcsp_subarray01.obsState
		assert obs_state == ObsState.ABORTED

		'''
		def test_configureScan_with_subarray_ready(self, midcsp_subarray01, midcsp_master):
		"""

csp-lmc-mid/tests/unit/midcspsubarray_unit_test.py

+153 −2

File changed.

Preview size limit exceeded, changes collapsed.