CT-66: Implemented Restart command (ADR-8). (3511e6b2) · Commits · Gianluca Marotta / csp-lmc-old

csp-lmc-common/csp_lmc_common/CspSubarray.py

+111 −3

Original line number	Diff line number	Diff line
		@@ -619,9 +619,10 @@ class CspSubarray(SKASubarray):
		if any(target_device._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.FAILED for device in device_list):
		target_device._failure_raised = True
		# reset the sub-element command execution flag
		for device in device_list:
		self.logger.info(target_device._sc_subarray_cmd_exec_state.keys())
		for device in target_device._sc_subarray_cmd_exec_state.keys():
		for cmd_name in target_device._sc_subarray_cmd_exec_state[device].keys():
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.IDLE

		self.logger.info("CspSubarray failure flag:{}".format(target_device._failure_raised))
		self.logger.info("CspSubarray timeout flag:{}".format(target_device._timeout_expired))
		if target_device._abort_obs_event.is_set():
		@@ -974,6 +975,86 @@ class CspSubarray(SKASubarray):
		target_device._last_executed_command = cmd_name
		self.logger.info("Abort ends with success")
		return target_device.abort_cmd_obj.succeeded()

		class RestartCommand(SKASubarray.RestartCommand):

		def do(self):
		self.logger.info("Call To Restart")
		device = self.target
		device_list = device._sc_subarray_assigned_fqdn
		if not any(device._sc_subarray_assigned_fqdn):
		# need to add a check also on PSTBeams belonging to subarray
		device_list = device._sc_subarray_fqdn
		# set all READY devices in IDLE
		for fqdn in device_list:
		proxy = device._sc_subarray_proxies[fqdn]
		if device._sc_subarray_obs_state[fqdn] == ObsState.READY:
		proxy.GoToIdle()
		for fqdn in device_list:
		proxy = device._sc_subarray_proxies[fqdn]
		self.logger.info("device {} obstate is {}".format(fqdn, ObsState(device._sc_subarray_obs_state[fqdn]).name))
		self.logger.info("lne(device) {}".format(len(device)))
		if device._sc_subarray_obs_state[fqdn] == ObsState.IDLE or len(device):
		if fqdn == device.CbfSubarray:
		self.logger.info("Issue command RemoveAllReceptors")
		proxy.command_inout_asynch("RemoveAllReceptors", device._cmd_ended_cb)
		if device._sc_subarray_obs_state[fqdn] in [ObsState.FAULT, ObsState.ABORTED]:
		proxy.command_inout_asynch("Restart", device._cmd_ended_cb)
		device._command_thread['restart'] = threading.Thread(target=self.restart_monitoring,
		name="Thread-Restart",
		args=(device_list,))
		device._command_thread['restart'].start()
		return (ResultCode.STARTED, "Restart command STARTED")

		def restart_monitoring(self, device_list):
		cmd_name = 'restart'
		target_device = self.target
		device_done = defaultdict(lambda:False)
		elapsed_time = 0
		starting_time = time.time()
		while True:
		for device in device_list:
		if device_done[device] == True:
		continue
		self.logger.info("device {} obs_state:{}".format(device, target_device._sc_subarray_obs_state[device]))
		if target_device._sc_subarray_obs_state[device] == ObsState.EMPTY:
		self.logger.info("Command {} ended with success on device {}.".format(cmd_name,
		device))
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.IDLE
		target_device._sc_subarray_cmd_progress[device][cmd_name] = 100
		# command success: step to next device
		device_done[device] = True
		# check if sub-element command ended throwing an exception: in this case the
		# 'cmd_ended_cb' callback is invoked.
		if target_device._sc_subarray_cmd_exec_state[device][cmd_name] == CmdExecState.FAILED or\
		target_device._sc_subarray_obs_state[device] == ObsState.FAULT:
		# execution ended for this sub-element, skip to the next one
		target_device._failure_raised = True
		device_done[device] = True
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.IDLE
		elapsed_time = time.time() - starting_time
		if elapsed_time > target_device._sc_subarray_cmd_duration_expected[device][cmd_name]:
		target_device._sc_subarray_cmd_exec_state[device][cmd_name] = CmdExecState.TIMEOUT
		target_device._timeout_expired = True
		device_done[device] = True

		if any(device_done.values()) and all(value == True for value in device_done.values()):
		self.logger.info("All devices have been handled!")
		break
		self.logger.info("Going to sleep")
		time.sleep(0.1)

		# end of the while loop
		# check for timeout/failure conditions on each sub-component
		if target_device._failure_raised or target_device._timeout_expired:
		return target_device.restart_cmd_obj.failed()

		if all(target_device._sc_subarray_obs_state[fqdn] == ObsState.EMPTY for fqdn in device_list):
		target_device._cmd_progress[cmd_name] = 100
		target_device._last_executed_command = cmd_name
		self.logger.info("Restart ends with success")
		return target_device.restart_cmd_obj.succeeded()

		'''
		class GoToIdleCommand(ActionCommand):
		"""
		@@ -1204,6 +1285,31 @@ class CspSubarray(SKASubarray):
		return (ResultCode.FAILED, "GoToIdle Command FAILED")
		return (ResultCode.OK, "GoToIdle command executed OK")

		class ResetCommand(SKASubarray.ResetCommand):
		def do(self):
		target_device = self.target
		device_list = target_device._sc_subarray_fqdn
		try:
		self.logger.info("Creating group for Reset {}".format(device_list))
		sc_group = tango.Group("ResetGroup")
		for device in device_list:
		sc_group.add(device)
		except Exception:
		self.logger.error("TANGO Group command failed")
		return (ResultCode.FAILED, "Reset Command FAILED")
		self.logger.info("Issue Reset")
		answers = sc_group.command_inout("Reset")
		for reply in answers:
		if reply.has_failed():
		for err in reply.get_err_stack():
		self.logger.error("device {}: {}-{}".format(reply.dev_name(), err.desc, err.reason))
		else:
		(result_code,msg) = reply.get_data()
		self.logger.error("device {}: {}".format(reply.dev_name(), msg))
		if any(target_device._sc_subarray_state[device]== DevState.FAULT for device in device_list):
		return (ResultCode.FAILED, "Reset Command FAILED")
		return (ResultCode.OK, "Reset command executed OK")

		# PROTECTED REGION ID(CspSubarray.class_variable) ENABLED START #
		# PROTECTED REGION END # // CspSubarray.class_variable
		# !! NOTE !!:
		@@ -1674,11 +1780,13 @@ class CspSubarray(SKASubarray):
		self.gotoidle_cmd_obj = self.GoToIdleCommand(*args)
		self.obsreset_cmd_obj = self.ObsResetCommand(*args)
		self.abort_cmd_obj = self.AbortCommand(*args)
		self.restart_cmd_obj = self.RestartCommand(*args)
		self.register_command_object("GoToIdle", self.GoToIdleCommand(*args))
		self.register_command_object("Configure", self.ConfigureCommand(*args))
		self.register_command_object("Scan", self.ScanCommand(*args))
		self.register_command_object("ObsReset", self.ObsResetCommand(*args))
		self.register_command_object("Abort", self.AbortCommand(*args))
		self.register_command_object("Restart", self.RestartCommand(*args))


		# ----------------

csp-lmc-mid/csp_lmc_mid/MidCspSubarrayBase.py

+0 −1

Original line number	Diff line number	Diff line
		@@ -602,7 +602,6 @@ class MidCspSubarrayBase(CspSubarray):
		args = (self, self.state_model, self.logger)
		self._addreceptors_cmd_obj = self.AddReceptorsCommand(*args)
		self._removereceptors_cmd_obj = self.RemoveReceptorsCommand(*args)
		self._removereceptors_cmd_obj = self.RemoveReceptorsCommand(*args)
		self.register_command_object("AddReceptors", self.AddReceptorsCommand(*args))
		self.register_command_object("RemoveReceptors", self.RemoveReceptorsCommand(*args))
		self.register_command_object("RemoveAllReceptors", self.RemoveAllReceptorsCommand(*args))

csp-lmc-mid/tests/MidCspSubarray_test.py

+73 −17

Original line number	Diff line number	Diff line
		@@ -63,8 +63,11 @@ class TestCspSubarray(TestBase):
		"""
		master_state = self.midcsp_master.State()
		if master_state == DevState.STANDBY:
		self.midcsp_master.On("")
		prober_subarray_state = Probe(self.midcsp_subarray01, "State", DevState.OFF, f"CSP Master not ON")
		argin =["mid_csp_cbf/sub_elt/master",]
		self.midcsp_master.On(argin)
		prober_master_state = Probe(self.midcsp_master, "State", DevState.ON, f"CSP Master not ON")
		Poller(4, 0.2).check(prober_master_state)
		prober_subarray_state = Probe(self.midcsp_subarray01, "State", DevState.ON, f"CSP Master not ON")
		Poller(4, 0.2).check(prober_subarray_state)
		subarray_state = self.midcsp_subarray01.State()
		if subarray_state == DevState.OFF:
		@@ -92,18 +95,10 @@ class TestCspSubarray(TestBase):
		self._release_all_receptors()

		def _setup_subarray_off(self):
		subarray_state = self.midcsp_subarray01.State()
		if subarray_state == DevState.OFF:
		return
		if subarray_state == DevState.ON:
		obs_state = self.midcsp_subarray01.obsState
		if obs_state == ObsState.READY:
		self._goto_idle()
		obs_state = self.midcsp_subarray01.obsState
		if obs_state == ObsState.IDLE:
		self._release_all_receptors()
		subarray_state = self.midcsp_subarray01.State()
		obs_state = self.midcsp_subarray01.obsState
		"""
		Set the subarray state to OFF-EMPTY
		"""
		self._setup_subarray()
		self.midcsp_subarray01.Off()
		prober_subarray_state = Probe(self.midcsp_subarray01, "State", DevState.OFF, f"CSP Master not ON")
		Poller(4, 0.2).check(prober_subarray_state)
		@@ -428,6 +423,7 @@ class TestCspSubarray(TestBase):
		f"Wrong CSP Subarray obsState")
		Poller(10, 0.2).check(prober_obs_state)

		'''
		def test_abort_scan(self):
		"""
		Test that a subarray is able to process the
		@@ -448,6 +444,7 @@ class TestCspSubarray(TestBase):
		prober_obs_state = Probe(self.midcsp_subarray01, 'obsState', ObsState.ABORTED,
		f"Wrong CSP Subarray obsState")
		Poller(10, 0.2).check(prober_obs_state)
		'''

		def test_invoke_configure_command_AFTER_gotoidle(self):
		self._configure_scan()
		@@ -511,6 +508,65 @@ class TestCspSubarray(TestBase):
		obs_state = self.midcsp_subarray01.obsState
		assert obs_state == ObsState.ABORTED

		'''
		def test_restart_AFTER_abort_with_subarray_idle(self):
		"""
		CSP Subarray invoke a RESTART after fault with
		subarray with allocated resources.
		"""
		# setup the test: Subarray DISABLE-IDLE
		self._setup_subarray()
		self._assign_receptors()
		self.midcsp_subarray01.Abort()
		prober_obs_state = Probe(self.midcsp_subarray01, "obsState", ObsState.ABORTED, f"CSP Subarray not EMPTY")
		Poller(4, 0.2).check(prober_obs_state)
		self.midcsp_subarray01.Restart()
		prober_obs_state = Probe(self.midcsp_subarray01, "obsState", ObsState.EMPTY, f"CSP Subarray not EMPTY")
		Poller(4, 0.2).check(prober_obs_state)
		assigned_receptors = self.midcsp_subarray01.assignedReceptors
		LOGGER.info("assigned_receptors:".format(assigned_receptors))

		def test_restart_AFTER_abort_with_subarray_ready(self):
		self._setup_subarray()
		self._assign_receptors()
		# configure the system with invalid json to send it in FAULT
		f = open(file_path + "/acceptance_tests/test_ConfigureScan_ADR4.json")
		LOGGER.info(f"Configuring CSP subarray01")
		self.midcsp_subarray01.Configure(f.read().replace("\n", ""))
		prober_obs_state = Probe(self.midcsp_subarray01, "obsState", ObsState.READY, f"CSP Subarray not READY")
		Poller(4, 0.2).check(prober_obs_state)
		self.midcsp_subarray01.Abort()
		prober_obs_state = Probe(self.midcsp_subarray01, "obsState", ObsState.ABORTED, f"CSP Subarray not ABORTED")
		Poller(4, 0.2).check(prober_obs_state)
		self.midcsp_subarray01.Restart()
		prober_obs_state = Probe(self.midcsp_subarray01, "obsState", ObsState.EMPTY, f"CSP Subarray not EMPTY")
		Poller(4, 0.2).check(prober_obs_state)
		'''

		def test_restart_AFTER_fault_with_subarray_idle(self):
		"""
		CSP Subarray invoke a RESTART after fault with
		subarray with allocated resources.
		"""
		# setup the test: Subarray ON-FAULT
		self._setup_subarray()
		self._assign_receptors()
		# configure the system with invalid json to send it in FAULT
		f = open(file_path + "/acceptance_tests/test_ConfigureScan_without_configID.json")
		self.midcsp_subarray01.Configure(f.read().replace("\n", ""))
		prober_obs_state = Probe(self.midcsp_subarray01, 'obsState',ObsState.FAULT, f"Failure flag is false")
		Poller(7, 0.2).check(prober_obs_state)
		obs_state = self.midcsp_subarray01.obsState
		state = self.midcsp_subarray01.State()
		LOGGER.info("CSPSubarray state: {}-{}".format(state, obs_state))
		# exercise the Subarray invoking Restart
		LOGGER.info(f"Invoke Restart command")
		self.midcsp_subarray01.Restart()
		# check
		# Subarray final ObsState EMPTY
		prober_obs_state = Probe(self.midcsp_subarray01, "obsState", ObsState.EMPTY, f"CSP Subarray not EMPTY")
		Poller(4, 0.2).check(prober_obs_state)

		'''
		def test_configureScan_with_subarray_ready(self, midcsp_subarray01, midcsp_master):
		"""