Commit 18581fb1 authored by vertighel's avatar vertighel
Browse files

Mako: one Vimba context per process; SIGUSR1 thread dump 

Every failed Mako connect entered the shared VmbSystem context without
leaving it (thousands per day with a camera unplugged), so VmbShutdown
never ran and VmbC aborted at exit with a boost mutex lock_error.

- mako: enter VmbSystem once per process, leave it once via atexit
  after closing the open cameras; __del__ now closes only its camera
- app: faulthandler on SIGUSR1 (`kill -USR1 <pid>`) dumps all thread
  stacks, to find the thread that hangs the shutdown after long uptime

Co-Authored-By: default avatarClaude Opus 5.5 <noreply@anthropic.com>
parent f3551cd7

noctua/app.py

+9 −0
Changes for noctua/app.py: 9 added lines, 0 removed lines.
Original line number Diff line number Diff line 
#!/usr/bin/env python3

# -*- coding: utf-8 -*-



# System modules

import faulthandler

import signal



# Third-party modules

import uvicorn

from quart import Quart
@@ -87,6 +91,11 @@ def run(): 
    # noctua-guider never call this, so they keep failing loudly.

    check.set_recoverable(True)



    # `kill -USR1 <pid>` prints the stack of every thread to stderr without

    # stopping the server: use it before Ctrl-C when the shutdown hangs,

    # to see which thread is stuck and where.

    faulthandler.register(signal.SIGUSR1, all_threads=True)



    for ws in ("websockets-sansio", "websockets"):

        config = uvicorn.Config(app, host="0.0.0.0", port=5533, ws=ws)

        try:

noctua/devices/mako.py

+38 −3
Changes for noctua/devices/mako.py: 38 added lines, 3 removed lines.
Original line number Diff line number Diff line
@@ -7,6 +7,7 @@ Camera ID is the IP address (e.g. 10.185.119.111). 
"""



# System modules

import atexit

import os

import threading

import time
@@ -51,6 +52,17 @@ class Mako(BaseDevice): 
    # lock serializes first-time connection attempts across all instances.

    _connect_lock = threading.Lock()



    # The VmbSystem context is entered once per process, on the first

    # connection attempt, and left once at exit (see shutdown_vmb).

    # VmbSystem counts its contexts and calls VmbShutdown only when the

    # count goes back to 0: entering it on every attempt without leaving

    # leaked one context per failed connect (thousands per day with a

    # camera unplugged), so VmbC was never shut down and aborted at exit

    # with a boost mutex error. Leaving it after each failure instead would

    # redo VmbStartup (~8 s) on every attempt.

    vmb_entered = False

    connected   = set()  # Mako instances with an open camera context



    def __init__(self, url):

        super().__init__(url)

        self.id   = url
@@ -89,7 +101,10 @@ class Mako(BaseDevice): 
                # threads queue up behind a connect that's already stuck.

                with check.timed_lock(Mako._connect_lock, "Mako connect"):

                    if self._cam is None:  # re-check: another thread may have connected while we waited

                        if not Mako.vmb_entered:

                            self.vmb.__enter__()

                            Mako.vmb_entered = True

                            atexit.register(Mako.shutdown_vmb)

                        try:

                            self._cam = self.vmb.get_camera_by_id(self.id)

                            self._cam.__enter__()
@@ -109,6 +124,7 @@ class Mako(BaseDevice): 
                            except Exception as e:

                                log.warning(f"Mako: packet size negotiation skipped: {e}")

                            self._set_throughput_limit(self._cam)

                            Mako.connected.add(self)

                        except Exception as e:

                            self._cam = None

                            msg = f"Mako connection failed: {e}"
@@ -142,17 +158,36 @@ class Mako(BaseDevice): 


        return True



    def __del__(self):

    def close(self):

        """Leave the camera context, if open."""



        if self._cam:

            try:

                self._cam.__exit__(None, None, None)

            except Exception:

                pass

        if self.vmb:

            self._cam = None

        Mako.connected.discard(self)



    @staticmethod

    def shutdown_vmb():

        """Close every open camera, then leave the shared VmbSystem context.



        Registered with atexit on the first connection attempt, so that

        VmbShutdown runs before the interpreter tears down VmbC.

        """



        for mako in list(Mako.connected):

            mako.close()

        if Mako.vmb_entered:

            try:

                self.vmb.__exit__(None, None, None)

                VmbSystem.get_instance().__exit__(None, None, None)

            except Exception:

                pass

            Mako.vmb_entered = False



    def __del__(self):

        self.close()





class Guider(Mako):