From 1136c55c5f3a116a3265fb57204f743aa2193c3f Mon Sep 17 00:00:00 2001 From: Jay Date: Wed, 4 Nov 2026 10:51:40 +0800 Subject: [PATCH 1/4] pyro: queue clients so one client can't interrupt another's measurement Previously, if two clients shared one QickSoc server, a second client's start_readout() silently aborted the first client's readout, and the first client's poll_data() loop could then consume the second client's data. start_server() now (by default) wraps the QickSoc class and its autoproxied members so every public method goes through a per-connection lock. A client owns the QICK while it has a call in progress and for lock_hold_time seconds after its last call; other clients' calls block until it is idle or disconnects. Internal calls from server-side threads are not locked. Adds soc.get_lock_status() and client_lock/lock_hold_time options. Co-Authored-By: Claude Opus 5.5 --- qick_lib/qick/pyro.py | 130 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 128 insertions(+), 2 deletions(-) diff --git a/qick_lib/qick/pyro.py b/qick_lib/qick/pyro.py index 17b4ed978..d6b960448 100644 --- a/qick_lib/qick/pyro.py +++ b/qick_lib/qick/pyro.py @@ -10,6 +10,115 @@ except: QickSoc = None +import threading, time, functools, inspect + +class ClientLock: + """Gives one Pyro client (connection) at a time exclusive use of the QICK. + + A client becomes the owner on its first call, and stays the owner while it has a call in progress + and for hold_time seconds after its last call (so a multi-round acquire() or a sweep loop is not interrupted). + Calls from other clients block (queue) until the owner is idle or disconnects. + Calls that don't come from a Pyro client (e.g. the streamer thread, or the QickSoc constructor) are never blocked. + """ + def __init__(self, hold_time=5.0): + self.hold_time = hold_time + # Condition uses an RLock, so nested calls within the same thread are fine + self._cond = threading.Condition() + self._owner = None + self._owner_addr = None + self._active = 0 + self._last = 0.0 + self._waiting = 0 + + def _owner_busy(self): + if self._owner is None: + return False + if self._active > 0: + return True + return time.time() - self._last < self.hold_time + + def enter(self, client, addr): + with self._cond: + if self._owner is not client: + self._waiting += 1 + try: + if self._owner_busy(): + print("QICK lock: client %s is waiting for client %s" % (addr, self._owner_addr)) + while self._owner_busy(): + # hold-time expiry doesn't notify, so wake up periodically + self._cond.wait(timeout=0.2) + finally: + self._waiting -= 1 + if self._owner_addr != addr: + print("QICK lock: client %s now owns the QICK" % (addr,)) + self._owner = client + self._owner_addr = addr + self._active = 0 + self._active += 1 + self._last = time.time() + + def exit(self, client): + with self._cond: + if self._owner is client: + self._active -= 1 + self._last = time.time() + self._cond.notify_all() + + def release(self, client): + """Called when a client disconnects.""" + with self._cond: + if self._owner is client: + print("QICK lock: owner %s disconnected, releasing" % (self._owner_addr,)) + self._owner = None + self._active = 0 + self._cond.notify_all() + + def status(self): + with self._cond: + busy = self._owner_busy() + return {'owner': self._owner_addr if busy else None, + 'waiting': self._waiting, + 'idle_time': time.time() - self._last if self._owner is not None else None} + +def _locked_method(func, lock): + @functools.wraps(func) + def wrapper(*args, **kwargs): + client = getattr(Pyro4.current_context, 'client', None) + if client is None: + # not a Pyro request (internal call from a server-side thread) + return func(*args, **kwargs) + lock.enter(client, getattr(Pyro4.current_context, 'client_sock_addr', None)) + try: + return func(*args, **kwargs) + finally: + lock.exit(client) + return wrapper + +# methods that clients may call without waiting for the lock +LOCK_EXEMPT = {'get_cfg', 'dump_cfg', 'get_lock_status'} + +def _make_locked_class(cls, lock): + """Make a subclass of cls where every public method goes through the ClientLock.""" + attrs = {} + for name in dir(cls): + if name.startswith('_') or name in LOCK_EXEMPT: + continue + raw = inspect.getattr_static(cls, name) + if inspect.isfunction(raw): + attrs[name] = _locked_method(raw, lock) + attrs['get_lock_status'] = lambda self: lock.status() + return type(cls.__name__, (cls,), attrs) + +class _LockingDaemon(Pyro4.Daemon): + """Pyro daemon that releases the QICK lock when the owning client disconnects.""" + def __init__(self, lock, **kwargs): + super().__init__(**kwargs) + self._qick_lock = lock + + def clientDisconnect(self, conn): + self._qick_lock.release(conn) + super().clientDisconnect(conn) + def start_nameserver(ns_host='0.0.0.0', ns_port=8888): """Starts a Pyro4 nameserver. @@ -27,7 +136,7 @@ def start_nameserver(ns_host='0.0.0.0', ns_port=8888): Pyro4.config.PICKLE_PROTOCOL_VERSION=4 Pyro4.naming.startNSloop(host=ns_host, port=ns_port) -def start_server(ns_host, ns_port=8888, proxy_name='myqick', soc_class=QickSoc, iface='eth0', **kwargs): +def start_server(ns_host, ns_port=8888, proxy_name='myqick', soc_class=QickSoc, iface='eth0', client_lock=True, lock_hold_time=5.0, **kwargs): """Initializes the QickSoc and starts a Pyro4 proxy server. Parameters @@ -44,6 +153,12 @@ def start_server(ns_host, ns_port=8888, proxy_name='myqick', soc_class=QickSoc, class to proxy, if you want to use a class other than QickSoc (e.g. if you need to use RFQickSoc) iface : str network interface whose address will be advertised - this should be the interface you expect to make Pyro connections on + client_lock : bool + if True, only one client at a time can use the QICK; calls from other clients wait until it is free + (the owner is free once it has been idle for lock_hold_time seconds, or disconnects) + lock_hold_time : float + how long (in seconds) a client keeps ownership after its last call; + must be longer than the client-side gaps between calls within a measurement (e.g. between rounds or sweep points) kwargs : optional named arguments any other options will be passed to the QickSoc constructor; see QickSoc documentation for details @@ -74,12 +189,23 @@ class to proxy, if you want to use a class other than QickSoc (e.g. if you need host = addrs_v4[0] if name==iface: break - daemon = Pyro4.Daemon(host=host) + if client_lock: + lock = ClientLock(hold_time=lock_hold_time) + daemon = _LockingDaemon(lock, host=host) + soc_class = _make_locked_class(soc_class, lock) + else: + daemon = Pyro4.Daemon(host=host) # if you want to use a different firmware image or set some initialization options, you would do that here soc = soc_class(**kwargs) print("initialized QICK") + if client_lock: + # the autoproxied members (tProc, streamer) can be called directly by clients, so lock them too + for obj in soc.autoproxy: + obj.__class__ = _make_locked_class(type(obj), lock) + print("client lock enabled (hold time %.1f s)" % (lock_hold_time,)) + # register the QickSoc in the daemon (so the daemon exposes the QickSoc over Pyro4) # and in the nameserver (so the client can find the QickSoc) ns.register(proxy_name, daemon.register(soc)) From 99f59ec09b82f5959276a9425b2b72164c587280 Mon Sep 17 00:00:00 2001 From: Jay Date: Wed, 4 Nov 2026 11:27:59 +0800 Subject: [PATCH 2/4] pyro: FIFO queue, wait_for_lock() and owner progress for the client lock Waiting clients can now see why they're waiting and for how long: - queued clients are served first-come-first-served - wait_for_lock(timeout) waits for at most `timeout` seconds while keeping the caller's place in the queue, and returns a status dict (granted, owner, queue position, owner progress), so clients can poll and stay cancellable - report_lock_progress(info) lets the owner publish its progress; the server-side readout progress of the current round is included as a fallback - get_lock_status() reports the same status without waiting Co-Authored-By: Claude Opus 5.5 --- qick_lib/qick/pyro.py | 172 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 142 insertions(+), 30 deletions(-) diff --git a/qick_lib/qick/pyro.py b/qick_lib/qick/pyro.py index d6b960448..b43113bc5 100644 --- a/qick_lib/qick/pyro.py +++ b/qick_lib/qick/pyro.py @@ -17,18 +17,26 @@ class ClientLock: A client becomes the owner on its first call, and stays the owner while it has a call in progress and for hold_time seconds after its last call (so a multi-round acquire() or a sweep loop is not interrupted). - Calls from other clients block (queue) until the owner is idle or disconnects. + Other clients wait in a first-come-first-served queue until the owner is idle or disconnects. + A client can either just make a call (it blocks until the QICK is free), + or poll wait_for_lock() to get status updates (owner, queue position, owner progress) while it waits. Calls that don't come from a Pyro client (e.g. the streamer thread, or the QickSoc constructor) are never blocked. """ - def __init__(self, hold_time=5.0): + def __init__(self, hold_time=5.0, queue_timeout=10.0): self.hold_time = hold_time + # a queued client that stops polling wait_for_lock() for this long loses its place + self.queue_timeout = queue_timeout + # optional callable returning server-side readout progress, set by start_server() + self.readout_progress = None # Condition uses an RLock, so nested calls within the same thread are fine self._cond = threading.Condition() self._owner = None - self._owner_addr = None self._active = 0 self._last = 0.0 - self._waiting = 0 + self._addrs = {} # client -> address, for clients that are queued or own the lock + self._queue = [] # waiting clients, oldest first + self._seen = {} # client -> last time it was seen waiting + self._progress = None # (client, info dict, time) reported by the owner def _owner_busy(self): if self._owner is None: @@ -37,23 +45,51 @@ def _owner_busy(self): return True return time.time() - self._last < self.hold_time + def _expire_queue(self): + now = time.time() + for client in list(self._queue): + if now - self._seen.get(client, 0) > self.queue_timeout: + print("QICK lock: client %s stopped waiting, removed from queue" % (self._addrs.get(client),)) + self._dequeue(client) + + def _dequeue(self, client): + if client in self._queue: + self._queue.remove(client) + self._seen.pop(client, None) + if client is not self._owner: + self._addrs.pop(client, None) + + def _try_take(self, client, addr): + """Wait in the queue for one step. Return True if the client now owns the lock.""" + if self._owner is client: + return True + if client not in self._queue: + self._queue.append(client) + self._addrs[client] = addr + if self._owner_busy(): + print("QICK lock: client %s is waiting for client %s" % (addr, self._addrs.get(self._owner))) + self._seen[client] = time.time() + self._expire_queue() + if self._owner_busy() or self._queue[0] is not client: + return False + self._queue.remove(client) + self._seen.pop(client, None) + if self._owner is not None: + self._addrs.pop(self._owner, None) + self._owner = client + self._addrs[client] = addr + self._active = 0 + self._last = time.time() + self._progress = None + print("QICK lock: client %s now owns the QICK" % (addr,)) + self._cond.notify_all() + return True + def enter(self, client, addr): with self._cond: - if self._owner is not client: - self._waiting += 1 - try: - if self._owner_busy(): - print("QICK lock: client %s is waiting for client %s" % (addr, self._owner_addr)) - while self._owner_busy(): - # hold-time expiry doesn't notify, so wake up periodically - self._cond.wait(timeout=0.2) - finally: - self._waiting -= 1 - if self._owner_addr != addr: - print("QICK lock: client %s now owns the QICK" % (addr,)) - self._owner = client - self._owner_addr = addr - self._active = 0 + # hold-time expiry doesn't notify, so wake up periodically + while not self._try_take(client, addr): + self._cond.wait(timeout=0.2) self._active += 1 self._last = time.time() @@ -64,30 +100,77 @@ def exit(self, client): self._last = time.time() self._cond.notify_all() + def wait(self, client, addr, timeout): + """Wait up to timeout seconds for the lock. Return a status dict; status['granted'] says if we got it. + A granted client keeps the lock for hold_time seconds, so it should start using the QICK right away.""" + end = time.time() + max(0.0, float(timeout)) + with self._cond: + while not self._try_take(client, addr): + remaining = end - time.time() + if remaining <= 0: + break + self._cond.wait(timeout=min(0.2, remaining)) + else: + self._last = time.time() + return self._status(client, addr) + + def report_progress(self, client, info): + """The owner can report its progress, so waiting clients can see it.""" + with self._cond: + if self._owner is client: + self._progress = (client, dict(info), time.time()) + def release(self, client): """Called when a client disconnects.""" with self._cond: if self._owner is client: - print("QICK lock: owner %s disconnected, releasing" % (self._owner_addr,)) + print("QICK lock: owner %s disconnected, releasing" % (self._addrs.get(client),)) self._owner = None self._active = 0 - self._cond.notify_all() + self._progress = None + self._dequeue(client) + self._addrs.pop(client, None) + self._cond.notify_all() - def status(self): + def _status(self, client, addr): + busy = self._owner_busy() + owner = self._owner if busy else None + progress = None + if owner is not None and self._progress is not None and self._progress[0] is owner: + progress = dict(self._progress[1], age=time.time() - self._progress[2]) + readout = None + if owner is not None and self.readout_progress is not None: + try: + readout = self.readout_progress() + except Exception: + readout = None + position = self._queue.index(client) + 1 if client in self._queue else None + return {'granted': owner is client or (owner is None and position is None), + 'you': addr, + 'owner': self._addrs.get(owner) if owner is not None else None, + 'owner_progress': progress, + 'owner_readout': readout, + 'position': position, + 'queue': [self._addrs.get(c) for c in self._queue], + 'idle_time': time.time() - self._last if owner is not None else None, + 'hold_time': self.hold_time} + + def status(self, client, addr): with self._cond: - busy = self._owner_busy() - return {'owner': self._owner_addr if busy else None, - 'waiting': self._waiting, - 'idle_time': time.time() - self._last if self._owner is not None else None} + return self._status(client, addr) + +def _pyro_caller(): + ctx = Pyro4.current_context + return getattr(ctx, 'client', None), getattr(ctx, 'client_sock_addr', None) def _locked_method(func, lock): @functools.wraps(func) def wrapper(*args, **kwargs): - client = getattr(Pyro4.current_context, 'client', None) + client, addr = _pyro_caller() if client is None: # not a Pyro request (internal call from a server-side thread) return func(*args, **kwargs) - lock.enter(client, getattr(Pyro4.current_context, 'client_sock_addr', None)) + lock.enter(client, addr) try: return func(*args, **kwargs) finally: @@ -95,7 +178,7 @@ def wrapper(*args, **kwargs): return wrapper # methods that clients may call without waiting for the lock -LOCK_EXEMPT = {'get_cfg', 'dump_cfg', 'get_lock_status'} +LOCK_EXEMPT = {'get_cfg', 'dump_cfg', 'get_lock_status', 'wait_for_lock', 'report_lock_progress'} def _make_locked_class(cls, lock): """Make a subclass of cls where every public method goes through the ClientLock.""" @@ -106,7 +189,28 @@ def _make_locked_class(cls, lock): raw = inspect.getattr_static(cls, name) if inspect.isfunction(raw): attrs[name] = _locked_method(raw, lock) - attrs['get_lock_status'] = lambda self: lock.status() + + def get_lock_status(self): + """Who owns the QICK, the queue, and the owner's progress. Never blocks.""" + return lock.status(*_pyro_caller()) + + def wait_for_lock(self, timeout=1.0): + """Wait up to timeout seconds to become the QICK owner, holding a place in the queue. + Returns the get_lock_status() dict; 'granted' is True once this client owns the QICK. + Call this repeatedly (at least every few seconds) to keep your place in the queue.""" + client, addr = _pyro_caller() + if client is None: + return {'granted': True} + return lock.wait(client, addr, timeout) + + def report_lock_progress(self, info): + """Owner-only: publish a progress dict (e.g. completed/total) for waiting clients to see.""" + client, _ = _pyro_caller() + if client is not None: + lock.report_progress(client, info) + + attrs.update(get_lock_status=get_lock_status, wait_for_lock=wait_for_lock, + report_lock_progress=report_lock_progress) return type(cls.__name__, (cls,), attrs) class _LockingDaemon(Pyro4.Daemon): @@ -204,6 +308,14 @@ class to proxy, if you want to use a class other than QickSoc (e.g. if you need # the autoproxied members (tProc, streamer) can be called directly by clients, so lock them too for obj in soc.autoproxy: obj.__class__ = _make_locked_class(type(obj), lock) + # let waiting clients see the progress of the current readout (shots in this round) + streamer = getattr(soc, 'streamer', None) + if streamer is not None: + def readout_progress(): + if streamer.done_flag.is_set(): + return None + return {'completed': int(streamer.count), 'total': int(streamer.total_count), 'unit': 'shot'} + lock.readout_progress = readout_progress print("client lock enabled (hold time %.1f s)" % (lock_hold_time,)) # register the QickSoc in the daemon (so the daemon exposes the QickSoc over Pyro4) From f05c97063c3e25be34237706311f860888979ecf Mon Sep 17 00:00:00 2001 From: Jay Date: Wed, 4 Nov 2026 12:23:17 +0800 Subject: [PATCH 3/4] pyro: identify lock clients by IP address, not connection Pyro4 drops and reopens a proxy's connection after a KeyboardInterrupt during a call, and autoproxied members (soc.streamer, soc.tproc) use their own connections. Keying the lock on the connection made a client's own cleanup calls after Ctrl-C (e.g. stop_tproc) queue behind itself for lock_hold_time, so they timed out. Ownership is now per peer IP; a client is released on disconnect only once all of its connections are closed. Co-Authored-By: Claude Opus 5.5 --- qick_lib/qick/pyro.py | 55 ++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/qick_lib/qick/pyro.py b/qick_lib/qick/pyro.py index b43113bc5..6df0bbf34 100644 --- a/qick_lib/qick/pyro.py +++ b/qick_lib/qick/pyro.py @@ -13,7 +13,12 @@ import threading, time, functools, inspect class ClientLock: - """Gives one Pyro client (connection) at a time exclusive use of the QICK. + """Gives one Pyro client at a time exclusive use of the QICK. + + A client is identified by its IP address, not by its connection: a proxy can open new connections + (Pyro4 drops and reopens the connection after a KeyboardInterrupt during a call), + and autoproxied members (soc.streamer, soc.tproc) each use their own connection. + So all connections from one PC share ownership, and different PCs queue for each other. A client becomes the owner on its first call, and stays the owner while it has a call in progress and for hold_time seconds after its last call (so a multi-round acquire() or a sweep loop is not interrupted). @@ -37,6 +42,14 @@ def __init__(self, hold_time=5.0, queue_timeout=10.0): self._queue = [] # waiting clients, oldest first self._seen = {} # client -> last time it was seen waiting self._progress = None # (client, info dict, time) reported by the owner + self._conns = {} # open Pyro connection -> client id + + def register(self, conn, addr): + """Map a Pyro connection to its client id (the peer IP address), remembering the connection.""" + client = addr[0] if isinstance(addr, (tuple, list)) and addr else conn + with self._cond: + self._conns[conn] = client + return client def _owner_busy(self): if self._owner is None: @@ -56,12 +69,12 @@ def _dequeue(self, client): if client in self._queue: self._queue.remove(client) self._seen.pop(client, None) - if client is not self._owner: + if client != self._owner: self._addrs.pop(client, None) def _try_take(self, client, addr): """Wait in the queue for one step. Return True if the client now owns the lock.""" - if self._owner is client: + if self._owner == client: return True if client not in self._queue: self._queue.append(client) @@ -70,7 +83,7 @@ def _try_take(self, client, addr): print("QICK lock: client %s is waiting for client %s" % (addr, self._addrs.get(self._owner))) self._seen[client] = time.time() self._expire_queue() - if self._owner_busy() or self._queue[0] is not client: + if self._owner_busy() or self._queue[0] != client: return False self._queue.remove(client) self._seen.pop(client, None) @@ -95,7 +108,7 @@ def enter(self, client, addr): def exit(self, client): with self._cond: - if self._owner is client: + if self._owner == client: self._active -= 1 self._last = time.time() self._cond.notify_all() @@ -117,13 +130,16 @@ def wait(self, client, addr, timeout): def report_progress(self, client, info): """The owner can report its progress, so waiting clients can see it.""" with self._cond: - if self._owner is client: + if self._owner == client: self._progress = (client, dict(info), time.time()) - def release(self, client): - """Called when a client disconnects.""" + def release(self, conn): + """Called when a connection closes. The client is released once it has no open connections left.""" with self._cond: - if self._owner is client: + client = self._conns.pop(conn, None) + if client is None or client in self._conns.values(): + return + if self._owner == client: print("QICK lock: owner %s disconnected, releasing" % (self._addrs.get(client),)) self._owner = None self._active = 0 @@ -136,7 +152,7 @@ def _status(self, client, addr): busy = self._owner_busy() owner = self._owner if busy else None progress = None - if owner is not None and self._progress is not None and self._progress[0] is owner: + if owner is not None and self._progress is not None and self._progress[0] == owner: progress = dict(self._progress[1], age=time.time() - self._progress[2]) readout = None if owner is not None and self.readout_progress is not None: @@ -145,7 +161,7 @@ def _status(self, client, addr): except Exception: readout = None position = self._queue.index(client) + 1 if client in self._queue else None - return {'granted': owner is client or (owner is None and position is None), + return {'granted': owner == client or (owner is None and position is None), 'you': addr, 'owner': self._addrs.get(owner) if owner is not None else None, 'owner_progress': progress, @@ -159,14 +175,21 @@ def status(self, client, addr): with self._cond: return self._status(client, addr) -def _pyro_caller(): +def _pyro_conn(): ctx = Pyro4.current_context return getattr(ctx, 'client', None), getattr(ctx, 'client_sock_addr', None) +def _caller(lock): + """(client id, address) of the Pyro caller, or (None, None) for internal calls.""" + conn, addr = _pyro_conn() + if conn is None: + return None, None + return lock.register(conn, addr), addr + def _locked_method(func, lock): @functools.wraps(func) def wrapper(*args, **kwargs): - client, addr = _pyro_caller() + client, addr = _caller(lock) if client is None: # not a Pyro request (internal call from a server-side thread) return func(*args, **kwargs) @@ -192,20 +215,20 @@ def _make_locked_class(cls, lock): def get_lock_status(self): """Who owns the QICK, the queue, and the owner's progress. Never blocks.""" - return lock.status(*_pyro_caller()) + return lock.status(*_caller(lock)) def wait_for_lock(self, timeout=1.0): """Wait up to timeout seconds to become the QICK owner, holding a place in the queue. Returns the get_lock_status() dict; 'granted' is True once this client owns the QICK. Call this repeatedly (at least every few seconds) to keep your place in the queue.""" - client, addr = _pyro_caller() + client, addr = _caller(lock) if client is None: return {'granted': True} return lock.wait(client, addr, timeout) def report_lock_progress(self, info): """Owner-only: publish a progress dict (e.g. completed/total) for waiting clients to see.""" - client, _ = _pyro_caller() + client, _ = _caller(lock) if client is not None: lock.report_progress(client, info) From a0272b49ef2a49030ee287790b9a2376b9706411 Mon Sep 17 00:00:00 2001 From: "QICK actions [bot]" <129547417+qickbot@users.noreply.github.com> Date: Wed, 4 Nov 2026 04:33:06 +0000 Subject: [PATCH 4/4] update version --- qick_lib/qick/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qick_lib/qick/VERSION b/qick_lib/qick/VERSION index 6bd7898f7..e9d6b949d 100644 --- a/qick_lib/qick/VERSION +++ b/qick_lib/qick/VERSION @@ -1 +1 @@ -0.2.442 +0.2.445