STX: survive post-readout blackouts instead of losing frames (f3551cd7) · Commits · Davide Ricci / noctua

noctua/devices/stx.py

+96 −16

Changes for noctua/devices/stx.py: 96 added lines, 16 removed lines.
Original line number	Diff line number	Diff line
		@@ -27,6 +27,66 @@ from ..utils.logger import log
		from .basedevice import BaseDevice


		class Throttle():
		"""
		Minimum spacing between HTTP commands sent to one physical STX.

		The lock only spaces out the start of the requests: it is not held
		during the request itself, so a long download does not block the
		status polling.
		"""

		def __init__(self, interval):
		"""
		Parameters
		----------
		interval : float
		Minimum time in seconds between two consecutive commands.
		"""

		self.interval = interval
		self.last_command = 0
		self.lock = threading.Lock()

		def wait(self):
		"""Sleep until the command interval has elapsed."""

		with self.lock:
		elapsed = time.time() - self.last_command
		if elapsed < self.interval:
		time.sleep(self.interval - elapsed)
		self.last_command = time.time()


		# One Throttle per camera address, shared by Camera, Guider and Ao
		# instances pointing to the same physical STX.
		throttles = {}
		throttles_lock = threading.Lock()


		def throttle_for(addr, interval=0.05):
		"""
		Return the Throttle shared by all the devices at the given address.

		Parameters
		----------
		addr : str
		The camera base URL.
		interval : float, optional
		Minimum time in seconds between two commands. Used only when the
		Throttle is created for the first time. Default is 0.05 (50 ms).

		Returns
		-------
		Throttle
		"""

		with throttles_lock:
		if addr not in throttles:
		throttles[addr] = Throttle(interval)
		return throttles[addr]


		class STX(BaseDevice):
		"""
		Base wrapper class for SBIG STX cameras.
		@@ -47,26 +107,21 @@ class STX(BaseDevice):
		self.url = url
		self.addr = self.url
		self.timeout = 3
		self._last_command_time = 0
		self._command_interval = 0.05 # 50 milliseconds
		# Guards _last_command_time: it's read/written by both the
		# background loop thread and request threads (get/put/matrix/
		# connection all call _wait_if_needed()) — without a lock, two
		# threads can race past the interval check together and both
		# fire a request within the same 50ms window.
		self._lock = threading.Lock()
		self.download_timeout = 60
		self.download_attempts = 3
		# Shared with the other devices (Camera, Guider, Ao) at the same
		# address: the background loop thread and the request threads
		# (get/put/matrix/connection/download) of all of them go through
		# the same lock, so the 50 ms spacing holds for the physical camera.
		self.throttle = throttle_for(self.addr)

		def _wait_if_needed(self):
		"""
		Ensures the 50ms command interval is respected
		between commands.
		between commands sent to this camera.
		"""

		with self._lock:
		elapsed = time.time() - self._last_command_time
		if elapsed < self._command_interval:
		time.sleep(self._command_interval - elapsed)
		self._last_command_time = time.time()
		self.throttle.wait()

		@property
		def connection(self):
		@@ -328,16 +383,41 @@ class Camera(STX):
		filepath : str, optional
		The local path to save the FITS file to.
		Defaults to the path configured in cameras.ini for this device.

		Returns
		-------
		str or None
		The path of the saved file, or None if the download failed.
		"""

		self.error = []

		if filepath is None:
		from ..config.constants import viewer_fits_path
		filepath = viewer_fits_path(getattr(self, '_viewer_key', None))

		# The STX web server can stop answering for 15-20 s right after
		# the readout: use a longer timeout than the status requests and
		# retry a few times before giving up.
		res = None
		for attempt in range(1, self.download_attempts + 1):
		self._wait_if_needed()
		log.debug(f"Getting original data (attempt {attempt})")
		try:
		res = requests.get(f"{self.addr}/{self.element}.FIT",
		timeout=self.download_timeout)
		res.raise_for_status()
		break
		except requests.exceptions.RequestException as e:
		log.warning(f"STX: download attempt {attempt}/"
		f"{self.download_attempts} failed: {e}")
		res = None

		log.debug(f"Getting original data")
		res = requests.get(f"{self.addr}/{self.element}.FIT")
		if res is None:
		msg = "STX: image download failed"
		log.error(msg)
		self.error.append(msg)
		return None
		log.debug(f"Got original data")

		from pathlib import Path

noctua/sequencer.py

+9 −5

Changes for noctua/sequencer.py: 9 added lines, 5 removed lines.
Original line number	Diff line number	Diff line
		@@ -170,21 +170,25 @@ class Sequencer():
		# Run the template given its parameters in the json.
		self.tpl.run(self.params)

		# log.exception() also writes the traceback, so the log shows
		# where the error was raised inside the template.
		except AttributeError as e:
		msg = f"SEQUENCER: Attribute Error in template '{template_name}'"
		log.error(msg)
		log.error(str(e))
		log.exception(msg)
		self.error.append(msg)
		self.error.append(str(e))

		except KeyError as e:
		msg = f"SEQUENCER: Parameter '{e}' is not defined for template '{template_name}'"
		log.error(msg)
		# A KeyError can come from anywhere inside the template
		# (e.g. a dict lookup on a device value), not only from a
		# missing OB parameter.
		msg = f"SEQUENCER: KeyError {e} in template '{template_name}'"
		log.exception(msg)
		self.error.append(msg)

		except Exception as e: # Catch other unexpected errors during template execution
		msg = f"SEQUENCER: Unexpected error during execution of template '{template_name}': {e}"
		log.error(msg)
		log.exception(msg)
		self.error.append(msg)
		self.error.append(str(e))
		finally:

noctua/templates/snapshot.py

+27 −6

Changes for noctua/templates/snapshot.py: 27 added lines, 6 removed lines.
Original line number	Diff line number	Diff line
		@@ -225,16 +225,37 @@ class Template(BaseTemplate):

		log.info(f"Downloading frame as {temp_fits}")

		# Checking if the image is ready to be downloaded
		while cam.ready != 1:
		log.debug(f"Downloading... (cam state {image_state[cam.ready]})")
		# Checking if the image is ready to be downloaded.
		# ready is None when the camera does not answer (the STX web
		# server can be silent for 15-20 s after the readout): keep
		# waiting up to ready_grace seconds, then try the download
		# anyway, since the frame is most likely in the buffer.
		ready_grace = 60
		ready_since = Time.now()
		ready = cam.ready
		while ready != 1:
		waited = (Time.now() - ready_since).sec.item()
		if ready is None:
		log.warning(f"Camera not answering while waiting for "
		f"the image ({waited:.1f}/{ready_grace}s)")
		else:
		log.debug(f"Downloading... (cam state {image_state.get(ready, ready)})")
		if waited > ready_grace:
		log.warning(f"Image not reported ready after "
		f"{ready_grace}s: trying the download anyway")
		break
		sleep(0.5)
		if self.check_pause_or_abort():
		log.warning(f"Buffer still {image_state[cam.ready]}")
		log.warning(f"Buffer still {image_state.get(ready, ready)}")
		return
		log.debug(f"Downloading... (cam state {image_state[cam.ready]})")
		ready = cam.ready
		log.debug(f"Downloading... (cam state {image_state.get(ready, ready)})")

		cam.download()
		if cam.download() is None:
		msg = f"Download failed: {cam.error}"
		log.error(msg)
		self.error.append(msg)
		return
		log.info(f"Elapsed {(Time.now() - now).sec.item():.2f}")

		##################################