diff --git a/changelog/70779.fixed.md b/changelog/70779.fixed.md new file mode 100644 index 000000000000..61eadbbf47ed --- /dev/null +++ b/changelog/70779.fixed.md @@ -0,0 +1 @@ +Align the cluster-ready restart test with the Raft log persistence semantics introduced by #70755. diff --git a/doc/ref/cache/all/index.rst b/doc/ref/cache/all/index.rst index 8047be56a334..9f6a070abf95 100644 --- a/doc/ref/cache/all/index.rst +++ b/doc/ref/cache/all/index.rst @@ -12,6 +12,7 @@ For understanding and usage of the cache modules see the :ref:`cache` topic. :toctree: :template: autosummary.rst.tmpl + _cluster_replicator consul etcd_cache localfs diff --git a/doc/ref/cache/all/salt.cache._cluster_replicator.rst b/doc/ref/cache/all/salt.cache._cluster_replicator.rst new file mode 100644 index 000000000000..f6133293215b --- /dev/null +++ b/doc/ref/cache/all/salt.cache._cluster_replicator.rst @@ -0,0 +1,5 @@ +salt.cache._cluster_replicator +============================== + +.. automodule:: salt.cache._cluster_replicator + :members: diff --git a/salt/cache/__init__.py b/salt/cache/__init__.py index da9f4fdd5576..76a0b2c977f3 100644 --- a/salt/cache/__init__.py +++ b/salt/cache/__init__.py @@ -19,6 +19,58 @@ log = logging.getLogger(__name__) +# Thread-local re-entrancy guard for cross-master cache replication. +# ``Cache.store`` / ``Cache.store_many`` consult this flag before +# emitting a ``cluster/runner/delegate_cache_write`` event: a peer-side +# handler (see ``salt/channel/server.py:_handle_delegate_cache_write``) +# applies incoming replication writes by calling ``cache.store`` again +# with the flag set, which must NOT re-broadcast or we'd build an +# infinite fan-out loop across the cluster. The ``originating_master`` +# check in the peer-side handler is the primary loop-safety guard; this +# thread-local is the second line of defense. +_REPLICATION_STATE = threading.local() + + +def _set_already_replicated(flag): + """Mark the current thread as applying a peer-originated write.""" + _REPLICATION_STATE.already_replicated = bool(flag) + + +def _is_already_replicated(): + return getattr(_REPLICATION_STATE, "already_replicated", False) + + +def _resolve_bank_routing(routing, bank): + """ + Longest-prefix-match lookup of a bank name against the routing map. + + Replication routing is keyed by top-level bank name (``jobs``, + ``cluster/consensus``, ...) but cache writes land on compound + sub-banks (``jobs/loads``, ``cluster/consensus//cluster/log``). + A plain ``routing.get(bank, default)`` therefore misses the common + case; this helper resolves by: + + 1. Exact-match the full bank name. + 2. Strip one ``/``-delimited segment at a time and retry. + 3. If no prefix matches, return ``"replicate"`` (the historical default). + + :param routing: Dict of ``{bank_prefix: "ring"|"replicate"|"none"}``, + as returned by ``salt.config.get_cluster_cache_routing``. + :param bank: Full bank name for the pending write. + :returns: Resolved mode string. + """ + if bank in routing: + return routing[bank] + parts = bank.split("/") + # Drop one trailing segment per iteration: a/b/c -> a/b -> a. + while len(parts) > 1: + parts.pop() + prefix = "/".join(parts) + if prefix in routing: + return routing[prefix] + return "replicate" + + def factory(opts, **kwargs): """ Creates and returns the cache class. @@ -76,6 +128,11 @@ def __init__(self, opts, cachedir=None, **kwargs): @cached_property def modules(self): + # Honor a pre-populated ``_modules`` dict so tests and + # non-loader-driven callers can inject a mock driver without + # touching the cached_property machinery. + if self._modules is not None: + return self._modules return salt.loader.cache(self.opts) @cached_property @@ -129,7 +186,7 @@ def cache(self, bank, key, fun, loop_fun=None, **kwargs): return data - def store(self, bank, key, data, expires=None): + def store(self, bank, key, data, expires=None, _already_replicated=False): """ Store data using the specified module @@ -149,13 +206,21 @@ def store(self, bank, key, data, expires=None): :param expires: how many seconds from now the data should be considered stale. + :param _already_replicated: + Internal flag used by the cluster-cache replication facade. + When True the local write proceeds but no cross-master + fan-out is performed. The peer-side handler passes this + when applying an incoming replication event so the same + write doesn't bounce back to every peer again. Not part + of the public API. + :raises SaltCacheError: Raises an exception if cache driver detected an error accessing data in the cache backend (auth, permissions, etc). """ fun = f"{self.driver}.store" try: - return self.modules[fun](bank, key, data, expires=expires, **self.kwargs) + result = self.modules[fun](bank, key, data, expires=expires, **self.kwargs) except TypeError: # if the backing store doesnt natively support expiry, we handle it as a fallback if expires: @@ -163,11 +228,203 @@ def store(self, bank, key, data, expires=None): seconds=expires ) expires_at = int(expires_at.timestamp()) - return self.modules[fun]( + result = self.modules[fun]( bank, key, {"data": data, "_expires": expires_at}, **self.kwargs ) else: - return self.modules[fun](bank, key, data, **self.kwargs) + result = self.modules[fun](bank, key, data, **self.kwargs) + # Cross-master replication fan-out at the facade. Guarded so + # non-clustered masters and shared-FS clusters pay no overhead. + # The peer-side re-dispatch arrives with + # ``_already_replicated=True`` and skips this block, breaking + # the loop (combined with the ``originating_master`` check in + # the handler). + if not _already_replicated and not _is_already_replicated(): + self._maybe_replicate_write(bank, key, data, expires) + return result + + def store_many(self, bank, items, expires=None, _already_replicated=False): + """ + Store multiple ``(key, data)`` pairs in a single bank. + + A thin wrapper over :meth:`store` that applies the same + cross-master replication semantics. The default + implementation loops; drivers that expose a bulk store + primitive can still accelerate the per-key hot path via the + underlying driver module (the facade intentionally keeps the + fan-out per-key so replication-ordering stays well-defined). + + :param bank: The bank name. + :param items: Iterable of ``(key, data)`` tuples. + :param expires: Optional seconds-from-now expiry applied to + every stored entry. + :param _already_replicated: Internal flag; see :meth:`store`. + """ + # Suppress replication during the inner per-key stores and + # emit a single batched replication event after the local + # writes land. This keeps the wire-cost proportional to + # batch count rather than batch size. + if isinstance(items, dict): + pairs = list(items.items()) + else: + pairs = list(items) + token_prev = _is_already_replicated() + _set_already_replicated(True) + try: + # Driver-native batch (e.g. ``mmap_cache.store_many``) wins + # one flock + batched fsync/msync across the whole batch. + # Drivers without a native batch fall back to the per-item + # loop so backwards compatibility is preserved. + fun = f"{self.driver}.store_many" + if fun in self.modules: + try: + self.modules[fun](bank, pairs, expires=expires, **self.kwargs) + except TypeError: + # Driver's store_many doesn't accept expires=; fall + # back to per-item store so expiry semantics stay + # uniform with the facade's envelope. + for key, data in pairs: + self.store( + bank, key, data, expires=expires, _already_replicated=True + ) + else: + for key, data in pairs: + self.store( + bank, key, data, expires=expires, _already_replicated=True + ) + finally: + _set_already_replicated(token_prev) + if not _already_replicated and not token_prev: + self._maybe_replicate_write_many(bank, pairs, expires) + + def _maybe_replicate_write(self, bank, key, data, expires): + """ + Fan a local cache write out to cluster peers. + + Consults ``cluster_cache_replication`` (master switch) and + ``cluster_cache_routing[bank]`` (per-bank routing) to decide + between three outcomes: + + - **replicate**: emit ``cluster/runner/delegate_cache_write`` + (handled by ``MasterPubServerChannel.publish_payload`` -> + ``publish_cache_write`` -> every peer). N-of-N fan-out. + - **ring**: delegate to the existing HashRing owner via the + ``cluster/runner/delegate_write`` event (1-of-N). Kept + for backward compatibility of the ``jobs`` bank. + - **skip**: non-clustered master, shared-FS mode, or an + unrecognised routing mode -- the local write stands alone. + + TODO: partial-peer failures are non-destructive today (the + originating master's local write already succeeded); catch-up + of a dropped peer is the state-sync expert's lane. See + ``agents/scratch/cache-replicate-primary.md``. + """ + opts = self.opts or {} + if not opts.get("cluster_id"): + return + # Isolated-FS is the only mode where peers can't see each + # other's local-store writes; shared-FS clusters are a no-op. + if not opts.get("cluster_isolated_filesystem"): + return + # Default-on when the cluster+isolated-FS preconditions are + # met; operators can force False. Mirrors the resolution in + # ``apply_master_config`` for callers that build opts without + # going through the config loader. + if opts.get("cluster_cache_replication") is False: + return + from salt.config import ( + get_cluster_cache_routing, # pylint: disable=import-outside-toplevel + ) + + routing = get_cluster_cache_routing(opts) + mode = _resolve_bank_routing(routing, bank) + if mode == "none": + # Explicit opt-out: bank is replicated by a dedicated + # transport (Raft for cluster/consensus, state-sync for + # cluster/state_sync). Fan-out here would overwrite the + # authoritative copy on peers. + return + if mode == "ring": + self._fire_ring_delegate(bank, key, data, expires) + elif mode == "replicate": + self._fire_replicate(bank, key, data, expires) + else: + log.warning( + "cluster-cache-replication: unknown routing %r for bank %r; " + "treating as 'replicate'", + mode, + bank, + ) + self._fire_replicate(bank, key, data, expires) + + def _maybe_replicate_write_many(self, bank, pairs, expires): + """ + Batched replication: emit ONE event carrying the whole list + (split into sub-batches of ``cluster_cache_replicate_batch_size`` + if the batch exceeds that threshold). + + The cluster + isolated-FS + routing gate is evaluated once + for the whole batch: all items share ``bank`` so routing is + uniform. This cuts per-write event overhead on hot bulk + paths (e.g. ``resource_grains`` convergence). + """ + if not pairs: + return + opts = self.opts or {} + if not opts.get("cluster_id"): + return + if not opts.get("cluster_isolated_filesystem"): + return + if opts.get("cluster_cache_replication") is False: + return + from salt.config import ( + get_cluster_cache_routing, # pylint: disable=import-outside-toplevel + ) + + routing = get_cluster_cache_routing(opts) + mode = _resolve_bank_routing(routing, bank) + if mode == "none": + return + if mode not in ("replicate", "ring"): + log.warning( + "cluster-cache-replication: unknown routing %r for bank %r; " + "treating as 'replicate'", + mode, + bank, + ) + mode = "replicate" + batch_size = int(opts.get("cluster_cache_replicate_batch_size", 1024) or 1024) + if batch_size <= 0: + batch_size = 1024 + from salt.cache._cluster_replicator import ( # pylint: disable=import-outside-toplevel + fire_delegate_cache_write_many, + fire_peer_cache_write_many, + ) + + fire = ( + fire_peer_cache_write_many + if mode == "replicate" + else fire_delegate_cache_write_many + ) + for start in range(0, len(pairs), batch_size): + chunk = pairs[start : start + batch_size] + fire(self.opts, bank, chunk, expires=expires) + + def _fire_replicate(self, bank, key, data, expires): + """Fan this write out to every peer master (see replicator module).""" + from salt.cache._cluster_replicator import ( # pylint: disable=import-outside-toplevel + fire_peer_cache_write, + ) + + fire_peer_cache_write(self.opts, bank, key, data, expires) + + def _fire_ring_delegate(self, bank, key, data, expires): + """Route this write to the HashRing owner (see replicator module).""" + from salt.cache._cluster_replicator import ( # pylint: disable=import-outside-toplevel + fire_delegate_cache_write, + ) + + fire_delegate_cache_write(self.opts, bank, key, data, expires) def fetch(self, bank, key): """ @@ -479,13 +736,18 @@ def fetch(self, bank, key): storage[(bank, key)] = [now, self.expire, data] return data - def store(self, bank, key, data, expires=None): + def store(self, bank, key, data, expires=None, _already_replicated=False): storage = self.storage with MemCache._lock: storage.pop((bank, key), None) # ``super().store`` calls the driver (disk / db); keep it off the # lock so concurrent stores don't serialise on returner I/O. - super().store(bank, key, data, expires=expires) + # Only forward the replication flag when it's set to True so + # legacy mock-based tests keep seeing the original call shape. + if _already_replicated: + super().store(bank, key, data, expires=expires, _already_replicated=True) + else: + super().store(bank, key, data, expires=expires) with MemCache._lock: if len(storage) >= self.max: if self.cleanup: diff --git a/salt/cache/_cluster_replicator.py b/salt/cache/_cluster_replicator.py new file mode 100644 index 000000000000..00816b3af029 --- /dev/null +++ b/salt/cache/_cluster_replicator.py @@ -0,0 +1,652 @@ +""" +Cross-master cache replication seams. + +Free-function entry points for both sides of the delegate-cache-write +flow so each side is testable without instantiating a channel class +or a Cache object. ``Cache.store`` and ``Cache.store_many`` emit via +:func:`fire_peer_cache_write` / :func:`fire_peer_cache_write_many` / +:func:`fire_delegate_cache_write`; +``MasterPubServerChannel._handle_delegate_cache_write`` delegates to +:func:`_handle_delegate_cache_write`. +""" + +import atexit +import concurrent.futures +import logging +import os +import threading + +import salt.cache + +log = logging.getLogger(__name__) + + +# Per-process shared Cache for the peer-apply hot path. +# +# ``_handle_delegate_cache_write`` previously built a fresh ``Cache(opts)`` +# on every incoming event. Under heavy peer-event load (hundreds/s) that +# forced every one of the publish daemon's ~32 executor threads to +# acquire the shared ``functools.cached_property`` RLock guarding +# ``Cache.modules``, which serialised them all through one critical +# section running ``salt.loader.cache(opts)``. Reusing one long-lived +# ``Cache`` moves the ``.modules`` lookup to a one-shot, after which +# the result lives in ``instance.__dict__`` and subsequent accesses +# bypass the descriptor's lock entirely. +# +# Keyed on ``id(opts)`` so a replacement opts dict triggers a rebuild. +# Driver-level writes (mmap_cache flock, localfs fcntl) remain +# thread-safe across concurrent ``store()`` calls on the shared +# instance; the only state ``Cache.store`` reads after the first +# ``.modules`` access is ``self.opts`` + ``self.kwargs`` (both +# immutable after construction). +_PEER_APPLY_CACHE_LOCK = threading.Lock() +_peer_apply_cache = (0, None) + + +def _get_peer_apply_cache(opts): + """ + Return a long-lived ``Cache`` for peer-apply writes. + + First call initialises; subsequent calls return the cached instance + without acquiring any lock (reading a Python-level reference is + atomic under the GIL). Rebuilds if ``id(opts)`` changes. + + :param opts: Master opts dict. + :rtype: salt.cache.Cache + """ + global _peer_apply_cache # pylint: disable=global-statement + opts_id = id(opts) + cur = _peer_apply_cache + if cur[0] == opts_id and cur[1] is not None: + return cur[1] + with _PEER_APPLY_CACHE_LOCK: + cur = _peer_apply_cache + if cur[0] != opts_id or cur[1] is None: + _peer_apply_cache = (opts_id, salt.cache.Cache(opts)) + return _peer_apply_cache[1] + + +# Per-(sock_dir, master-id, pid) cached MasterEvent handle. +# +# ``Cache.store`` / ``Cache.store_many`` on a clustered master fire a +# cluster-runner event on every write. Constructing a fresh +# ``MasterEvent`` (connect + fire + close) per write was measurably +# expensive at scale (200K-resource register adds ~70s per master). +# The handle is safe to reuse for the fire-only direction +# (``listen=False``): it just pushes payloads at the master event bus. +# +# Fork-safety: when a worker process forks, the child inherits the +# dict but the underlying file descriptors in the parent's handle are +# stale. We key on ``os.getpid()`` so each process constructs its +# own handle lazily; the dict entry for the parent's pid becomes +# unused (but harmless) in the child. +_EVENT_HANDLE_CACHE = {} +_EVENT_HANDLE_LOCK = threading.Lock() + + +# Per-minion single-worker executor for peer-side register apply. +# +# The publish daemon dispatches each incoming +# ``cluster/peer/delegate-resource-register-write`` event into a +# ThreadPoolExecutor via ``run_in_executor(executor, ...)``. The +# default (shared) ``run_in_executor(None, ...)`` gives no FIFO +# ordering guarantee, so a fast ``chunk_final=True`` arriving while +# an earlier non-final chunk is still queued could run the prune +# pass against an incomplete pending window and wipe valid SRNs. +# +# Fix shape: route every event for a given ``minion_id`` through a +# dedicated ``concurrent.futures.ThreadPoolExecutor(max_workers=1)``. +# That executor has an internal FIFO work queue, so successive +# submissions for the same minion run in submission order with no +# inter-chunk lock contention. Different minions each get their own +# executor and run in parallel. +# +# The inner ``_pending_registers_lock`` in +# :meth:`ResourceRegistry.register_minion_chunk` still guards the +# ``_pending_registers`` dict against interleaving with originator +# MWorker paths that also write through the shared registry +# singleton in the same process. +# +# Replaces the previous per-``(minion_id, chunk_scope)`` ``threading.Lock`` +# registry: that design worked but produced heavy contention during +# high-volume single-minion register storms because every +# ``_handle_delegate_resource_register_write`` thread blocked on the +# same contended lock. Executor FIFO sidesteps the contention entirely. +# +# Future refactor: the per-minion work here could be an ``asyncio.Queue`` +# fed by an ``asyncio.Task`` consumer, with each individual mmap write +# wrapped in ``asyncio.to_thread`` (or ``loop.run_in_executor(None, …)``) +# since the underlying ``MmapCache`` write path is synchronous and +# cannot yield. Benefits over the current thread-per-minion design: +# +# * lower per-minion memory (task object vs. OS thread stack) — useful +# as deployments scale their minion fleet +# * cleaner cancellation on daemon shutdown (``task.cancel()`` instead +# of best-effort ``ThreadPoolExecutor.shutdown`` from atexit) +# * one shared worker pool for the sync mmap calls instead of a thread +# per minion +# +# Not pursued here because the realistic swap still ends up offloading +# the mmap calls to a thread via ``to_thread`` — it moves the thread +# cost, it does not eliminate it — and the current design is simple +# and correct. Revisit when per-minion thread count becomes observable +# in master RSS or shutdown latency. +_PEER_APPLY_EXECUTORS = {} +_PEER_APPLY_EXECUTORS_GUARD = threading.Lock() + + +def _get_peer_apply_executor(minion_id): + """ + Return the dedicated single-worker executor for peer-side register + apply for ``minion_id``. + + Created lazily on first use. The executor holds one thread for + the lifetime of the publish-daemon process; per-minion overhead + is bounded by ``N_minions`` registered within the process + lifetime (small compared to the resource-registry itself). + + The executor's internal work queue gives natural FIFO ordering + for every event dispatched through this function, which is the + correctness property that preserves chunk ordering within a + minion's registration window. + """ + with _PEER_APPLY_EXECUTORS_GUARD: + executor = _PEER_APPLY_EXECUTORS.get(minion_id) + if executor is None: + executor = concurrent.futures.ThreadPoolExecutor( + max_workers=1, + thread_name_prefix=f"peer-apply-{minion_id}", + ) + _PEER_APPLY_EXECUTORS[minion_id] = executor + return executor + + +def _shutdown_peer_apply_executors(): + """Best-effort teardown of every per-minion apply executor.""" + with _PEER_APPLY_EXECUTORS_GUARD: + execs = list(_PEER_APPLY_EXECUTORS.values()) + _PEER_APPLY_EXECUTORS.clear() + for ex in execs: + try: + ex.shutdown(wait=False, cancel_futures=True) + except Exception: # pylint: disable=broad-except + # Interpreter shutdown can race the shutdown path; swallow. + pass + + +atexit.register(_shutdown_peer_apply_executors) + + +def _handle_cache_key(opts): + """Compose a (sock_dir, master_id, pid) key for the handle cache.""" + return (opts.get("sock_dir"), opts.get("id"), os.getpid()) + + +def _get_event_handle(opts): + """ + Return a cached ``MasterEvent`` for firing replication events. + + The handle is created lazily on first use in each process (so + worker forks naturally pick up their own) and reused across + subsequent calls in the same pid. Teardown is handled by + ``atexit`` (best-effort -- replication events are not critical + on shutdown). + """ + import salt.utils.event # pylint: disable=import-outside-toplevel + + key = _handle_cache_key(opts) + handle = _EVENT_HANDLE_CACHE.get(key) + if handle is not None: + return handle + with _EVENT_HANDLE_LOCK: + handle = _EVENT_HANDLE_CACHE.get(key) + if handle is not None: + return handle + handle = salt.utils.event.get_event( + "master", + sock_dir=opts.get("sock_dir"), + opts=opts, + listen=False, + ) + _EVENT_HANDLE_CACHE[key] = handle + return handle + + +def _close_cached_handles(): + """Best-effort teardown of every cached MasterEvent on exit.""" + with _EVENT_HANDLE_LOCK: + handles = list(_EVENT_HANDLE_CACHE.values()) + _EVENT_HANDLE_CACHE.clear() + for handle in handles: + try: + handle.destroy() + except Exception: # pylint: disable=broad-except + # Interpreter shutdown can race the destroy path; swallow. + pass + + +atexit.register(_close_cached_handles) + + +def _fire_payload(opts, payload, tag="cluster/runner/delegate_cache_write"): + """ + Fire ``payload`` on the master event bus using a pid-cached handle. + + Used by both the per-write and batched replication paths. The + handle is reused across calls; see :func:`_get_event_handle`. + """ + try: + handle = _get_event_handle(opts) + handle.fire_event(payload, tag) + except Exception: # pylint: disable=broad-except + # Invalidate the cached handle on failure so the next call + # re-opens the socket. The underlying ``get_event`` socket + # may have been closed out from under us (e.g. master + # restart) and a stale handle will keep failing until + # reset. + key_cache = _handle_cache_key(opts) + with _EVENT_HANDLE_LOCK: + stale = _EVENT_HANDLE_CACHE.pop(key_cache, None) + if stale is not None: + try: + stale.destroy() + except Exception: # pylint: disable=broad-except + pass + log.exception( + "cluster-cache-replication: failed to fire %s event (payload keys=%s)", + tag, + list(payload) if isinstance(payload, dict) else type(payload).__name__, + ) + + +def _fire_event(opts, write_kind, bank, key, data, expires): + """ + Common implementation: fire ``cluster/runner/delegate_cache_write`` + with the given write_kind. The master's publish daemon intercepts + the event and either fans it out to every peer (``replicate``) or + routes it to the HashRing owner (``ring``). + + The underlying ``MasterEvent`` handle is cached per + ``(sock_dir, opts["id"], pid)`` so hot-path writes + (``cache.store("grains", ...)``) avoid the connect + close cost + on every call. See :func:`_get_event_handle`. + """ + payload = { + "bank": bank, + "key": key, + "data": data, + "expires": expires, + "write_kind": write_kind, + "originating_master": opts.get("id"), + } + _fire_payload(opts, payload) + + +def fire_peer_cache_write(opts, bank, key, data, expires=None): + """ + Fan a cache write out to every peer master (N-of-N replication). + + Called by :meth:`salt.cache.Cache.store` and + :meth:`salt.cache.Cache.store_many` when the per-bank routing + resolves to ``"replicate"`` (the default). + + This is the single seam tests patch to observe fan-out intent + without a real event bus. + """ + _fire_event(opts, "replicate", bank, key, data, expires) + + +def fire_peer_cache_write_many(opts, bank, items, expires=None, write_kind="replicate"): + """ + Batched N-of-N replication: fan a list of ``(key, data)`` pairs + out to every peer master in a single event. + + The gate (cluster + isolated-FS + per-bank routing) is evaluated + once by the caller -- all items share the same bank so routing + is uniform. This cuts the per-write event cost on hot batched + paths like ``cache.store_many`` for resource grains. + + :param opts: Master opts dict. + :param bank: The shared bank name. + :param items: Iterable of ``(key, data)`` tuples. + :param expires: Optional seconds-from-now expiry applied to + every entry in the batch. + :param write_kind: ``"replicate"`` (default, N-of-N fan-out) or + ``"ring"`` (1-of-N; see + :func:`fire_delegate_cache_write_many`). Used to let the + receiver dispatch via the same ``_run_delegate_cache_write`` + path as the single-item events. + """ + pairs = list(items) + if not pairs: + return + payload = { + "bank": bank, + "items": [[key, data] for key, data in pairs], + "expires": expires, + "write_kind": write_kind, + "batch": True, + "originating_master": opts.get("id"), + } + _fire_payload(opts, payload) + + +def fire_delegate_cache_write(opts, bank, key, data, expires=None): + """ + Route a cache write to a single HashRing owner (1-of-N delegation). + + Called when the per-bank routing resolves to ``"ring"`` (e.g. the + ``jobs`` bank by default, for backward compat with the existing + ring-sharded job returner). + """ + _fire_event(opts, "ring", bank, key, data, expires) + + +def fire_delegate_cache_write_many(opts, bank, items, expires=None): + """Batched 1-of-N ring-delegation variant of :func:`fire_peer_cache_write_many`.""" + fire_peer_cache_write_many(opts, bank, items, expires=expires, write_kind="ring") + + +def _partition_resources_by_ring(ring, resources): + """ + Split a ``{resource_type: [resource_id, ...]}`` dict by ring owner. + + Returns ``{owner_node_id: {resource_type: [resource_id, ...]}}``. + The ring is consulted per-SRN (``":"``) via + :meth:`HashRing.get_owner`; SRNs whose owner cannot be resolved + (empty ring) are returned under the ``None`` key so the caller + can fall back to broadcast for them. + """ + import salt.utils.resource_registry # pylint: disable=import-outside-toplevel + + buckets = {} + for rtype, rids in (resources or {}).items(): + for rid in rids or (): + srn = salt.utils.resource_registry.resource_index_srn_key(rtype, rid) + owner = ring.get_owner(srn) + per_owner = buckets.setdefault(owner, {}) + per_owner.setdefault(rtype, []).append(rid) + return buckets + + +def fire_resource_register_write( + opts, + minion_id, + resources, + chunk_final=True, + chunk_scope=None, + chunk_index=None, +): + """ + Fire a resource-registry write event for the publish daemon to + intercept and dispatch. + + The MWorker handling ``_register_resources`` runs this to hand the + register off to the publish daemon. The daemon + (:meth:`salt.channel.server.MasterPubServerChannel._run_delegate_resource_register_write`) + is the single place that owns the ring-vs-broadcast decision, the + local-shard mmap write, and the targeted-vs-fan-out peer publish + -- because the ``resource_index`` ring state only lives in the + daemon process (its per-ring Raft groups aren't visible across the + MWorker fork boundary). + + ``chunk_final`` + ``chunk_scope`` + ``chunk_index`` carry chunked- + register metadata through to peers and the local publish-daemon + writer so a multi-chunk register preserves additive mid-stream + puts and defers the stale-SRN prune to the terminal chunk. The + ``chunk_index`` field additionally lets peer masters detect and + buffer out-of-order final-chunk arrivals (publish-daemon + ``run_in_executor`` has no FIFO guarantee across chunks). Legacy + single-shot callers can omit all three fields and get pre-chunk + semantics (full replace, no gap check). + + :param opts: Master opts dict (must contain ``id``). + :param minion_id: The reporting minion. + :param resources: ``{resource_type: [resource_id, ...]}`` -- the + exact dict that was passed to the local + ``update_resource_index_chunk`` call. + :param chunk_final: True iff this chunk ends a register window. + :param chunk_scope: ``None``/``"__all__"`` for a legacy single- + shot register, or a type name for a scoped + chunked register. + :param chunk_index: 0-based monotonic index of this chunk within + the register window. ``None`` for legacy + single-shot callers. + """ + payload = { + "minion_id": minion_id, + "resources": resources, + "originating_master": opts.get("id"), + "chunk_final": chunk_final, + "chunk_scope": chunk_scope or "__all__", + } + if chunk_index is not None: + payload["chunk_index"] = chunk_index + _fire_payload( + opts, + payload, + tag="cluster/runner/delegate_resource_register_write", + ) + + +def _get_resource_index_ring(): + """ + Return the local ``resource_index`` HashRing, or ``None`` if the + cluster module isn't importable (standalone test fixtures). + """ + try: + import salt.cluster.ring_membership # pylint: disable=import-outside-toplevel + import salt.utils.resource_registry # pylint: disable=import-outside-toplevel + except Exception: # pylint: disable=broad-except + return None + try: + return salt.cluster.ring_membership.get_ring( + salt.utils.resource_registry.RESOURCE_INDEX_RING_NAME + ) + except Exception: # pylint: disable=broad-except + return None + + +def partition_resources_for_dispatch(opts, resources): + """ + Split ``resources`` into ``(local_bucket, peer_buckets, ring_active)`` + for the publish-daemon dispatcher. + + Used by + :meth:`salt.channel.server.MasterPubServerChannel._run_delegate_resource_register_write` + inside the publish daemon process where the ``resource_index`` + ring is populated. When the ring is empty, returns the full + ``resources`` as the "broadcast everyone" bucket and no peer + buckets. When populated, splits by :meth:`HashRing.get_owner`. + + :param opts: Master opts dict. + :param resources: ``{resource_type: [resource_id, ...]}``. + :returns: ``(local_bucket, peer_buckets, ring_active)`` where + ``peer_buckets`` is ``{owner_node_id: resources_dict}`` + for each remote owner, and ``ring_active`` is a + ``bool`` indicating ring routing is in effect. + """ + import salt.cluster # pylint: disable=import-outside-toplevel + + ring = _get_resource_index_ring() + if ring is None or not ring.nodes(): + return resources, {}, False + self_node = salt.cluster.node_id(opts) if opts.get("id") else None + buckets = _partition_resources_by_ring(ring, resources) + local = dict(buckets.get(self_node, {})) + # SRNs whose owner didn't resolve land locally so they're not lost + # to a fencepost in the ring state. + for rtype, rids in buckets.get(None, {}).items(): + local.setdefault(rtype, []).extend(rids) + peers = { + owner: owner_resources + for owner, owner_resources in buckets.items() + if owner is not None and owner != self_node + } + return local, peers, True + + +def _handle_delegate_resource_register_write(opts, payload): + """ + Apply a peer-originated resource-registry write locally. + + Peer-side counterpart to :func:`fire_resource_register_write`. + Same loop-safety shape as :func:`_handle_delegate_cache_write`: + + 1. Drop the event if ``originating_master`` equals this master's + ``opts["id"]`` -- bus bounce-back must not become a write. + 2. Apply via :func:`salt.utils.minions.update_resource_index_chunk` + so the local write lands through the same chunked code path a + minion-originated register would take. Non-final chunks + additively ``put_many`` without pruning; only the terminal + chunk runs the stale-SRN delete pass scoped to ``chunk_scope``. + + Idempotency: the chunked API's per-minion accumulator keys by + scope, so duplicate deliveries converge to the same state (the + same SRN set is re-inserted into the pending window, and a + duplicate terminal chunk's delete-diff is a no-op). + + :param opts: Master opts dict (must contain ``id``). + :param payload: Decoded event payload with keys + ``originating_master``, ``minion_id``, ``resources``, + and (chunked registers only) + ``chunk_final``/``chunk_scope``. Missing chunk + fields default to ``True``/``"__all__"`` for + back-compat with legacy single-shot payloads. + :returns: ``True`` if the register was applied, ``False`` + if dropped (loop-guard or malformed payload). + """ + if payload.get("originating_master") == opts.get("id"): + return False + minion_id = payload.get("minion_id") + if not minion_id: + log.warning( + "resource-replicate: incoming event missing minion_id; dropping %r", + payload, + ) + return False + resources = payload.get("resources") + if resources is None: + log.warning( + "resource-replicate: incoming event missing resources; dropping %r", + payload, + ) + return False + chunk_final = payload.get("chunk_final", True) + chunk_scope = payload.get("chunk_scope") or "__all__" + chunk_index = payload.get("chunk_index") + try: + import salt.utils.minions # pylint: disable=import-outside-toplevel + + # Ordering invariant: this call expects the publish daemon + # to have dispatched through :func:`_get_peer_apply_executor` + # (a per-minion, single-worker ThreadPoolExecutor) so every + # event for ``minion_id`` runs in submission order. The + # inner ``_pending_registers_lock`` in + # :meth:`ResourceRegistry.register_minion_chunk` still + # protects the pending-window dict against the originator + # MWorker apply path that shares the same registry singleton. + # The ``chunk_index`` gap check handles any residual out-of- + # order case (e.g. legacy callers bypassing the executor). + salt.utils.minions.update_resource_index_chunk( + opts, + minion_id, + resources, + chunk_final, + chunk_scope=chunk_scope, + chunk_index=chunk_index, + ) + return True + except Exception: # pylint: disable=broad-except + log.exception( + "resource-replicate: failed to apply replicated register for %s", + minion_id, + ) + return False + + +def _handle_delegate_cache_write(opts, payload): + """ + Apply a peer-originated cache write locally. + + Handles both the per-item and batched (``batch: True``, + ``items: [[key, data], ...]``) payload shapes. + + Two loop-safety guards: + + 1. Drop the event if ``originating_master`` matches this master's + ``opts["id"]`` -- bus bounce-back must not become a write. + 2. Pass ``_already_replicated=True`` to :meth:`salt.cache.Cache.store` + so the peer-side write does NOT re-broadcast the replication + event to every peer again. + + Idempotency comes from the cache driver's replace semantics -- + duplicate deliveries converge to the same final state. + + :param opts: Master opts dict (must contain ``id``). + :param payload: Decoded event payload with keys ``originating_master``, + ``bank``, ``key``, ``data``, and optional ``expires``; + or with ``batch: True`` and ``items: [[k,v], ...]``. + :returns: ``True`` if at least one write was applied, ``False`` + if dropped (loop-guard or malformed payload). + """ + if payload.get("originating_master") == opts.get("id"): + return False + bank = payload.get("bank") + if not bank: + log.warning( + "cache-replicate: incoming event missing bank; dropping %r", payload + ) + return False + expires = payload.get("expires") + try: + cache = _get_peer_apply_cache(opts) + except Exception: # pylint: disable=broad-except + log.exception( + "cache-replicate: failed to obtain shared Cache for replication apply" + ) + return False + if payload.get("batch"): + items = payload.get("items") or [] + if not items: + log.warning( + "cache-replicate: batched event with empty items list; dropping" + ) + return False + applied_any = False + for entry in items: + try: + key, data = entry[0], entry[1] + except (KeyError, IndexError, TypeError): + log.warning( + "cache-replicate: malformed batch entry %r; skipping", entry + ) + continue + try: + cache.store(bank, key, data, expires=expires, _already_replicated=True) + applied_any = True + except Exception: # pylint: disable=broad-except + log.exception( + "cache-replicate: failed to apply replicated write for %s/%s", + bank, + key, + ) + return applied_any + key = payload.get("key") + if not key: + log.warning( + "cache-replicate: incoming event missing bank/key; dropping %r", + payload, + ) + return False + data = payload.get("data") + try: + cache.store(bank, key, data, expires=expires, _already_replicated=True) + return True + except Exception: # pylint: disable=broad-except + log.exception( + "cache-replicate: failed to apply replicated write for %s/%s", + bank, + key, + ) + return False diff --git a/salt/cache/mmap_cache.py b/salt/cache/mmap_cache.py index c0be3ef7b15c..457d55431a04 100644 --- a/salt/cache/mmap_cache.py +++ b/salt/cache/mmap_cache.py @@ -201,6 +201,52 @@ def store(bank, key, data, cachedir, **kwargs): ) +def store_many(bank, items, cachedir, **kwargs): + """ + Bulk-store many ``(key, data)`` pairs under *bank* in a single + :meth:`~salt.utils.mmap_cache.MmapCache.put_many` call. + + Equivalent in effect to looping ``store(bank, k, v)`` over *items*, + but pays the flock + fsync + msync cost once for the whole batch + instead of once per entry. Required for batched hot paths like + the master's resource-grain register handler to keep up with + 5 concurrent minion registers. + + :param bank: cache bank (same semantics as :func:`store`). + :param items: iterable of ``(key, data)`` pairs. May be a dict, in + which case ``dict.items()`` is used. + :param cachedir: driver cachedir (injected by the facade's + :attr:`salt.cache.Cache.kwargs`). + + :raises SaltCacheError: if any value fails to msgpack-serialise + (the batch is aborted before the ``put_many`` call and no + items are written) or if the underlying ``put_many`` reports + a failure. + """ + if isinstance(items, dict): + items = items.items() + + packed = [] + for key, data in items: + try: + raw = msgpack.packb(data, **_PACK_OPTS) + except Exception as exc: # pylint: disable=broad-except + raise SaltCacheError( + f"Failed to serialise cache data for bank={bank!r} key={key!r}: {exc}" + ) + packed.append((key, raw)) + + if not packed: + return + + cache = _get_cache(bank, cachedir) + if not cache.put_many(packed): + raise SaltCacheError( + f"Failed to bulk-write mmap cache entries bank={bank!r} " + f"(count={len(packed)})" + ) + + def fetch(bank, key, cachedir, **kwargs): """ Return the deserialised value for *bank*/*key*, or ``{}`` if not found. diff --git a/salt/channel/server.py b/salt/channel/server.py index 2ffe72137bab..9faa4b4a6dcb 100644 --- a/salt/channel/server.py +++ b/salt/channel/server.py @@ -103,7 +103,7 @@ def _transport_has_builtin_router(transport): ZeroMQ's ``pre_fork`` adds ``zmq_device_pooled`` for that purpose, so :class:`PoolRoutingChannel` does not need to spawn its own router. Other transports (TCP, WebSockets) bind the external socket but rely on - a separate process to serve it — :class:`PoolRoutingChannel.pre_fork` + a separate process to serve it -- :class:`PoolRoutingChannel.pre_fork` spawns ``_run_pool_router`` for that case. """ module = getattr(transport.__class__, "__module__", "") or "" @@ -520,7 +520,7 @@ async def handle_message(self, payload): # Block non-_auth requests until this node is a committed Raft voter. if not _cluster_is_ready(self.opts): log.debug( - "Cluster not ready yet — deferring request from %s", + "Cluster not ready yet -- deferring request from %s", payload.get("load", {}).get("id", "unknown"), ) return {"enc": "clear", "load": {"ret": False, "cluster_retry": True}} @@ -775,7 +775,7 @@ class PoolRoutingChannel: This class is returned by :meth:`ReqServerChannel.factory` when ``worker_pools_enabled=True`` (the default). It is mutually exclusive - with the plain :class:`ReqServerChannel` — only one of the two is ever + with the plain :class:`ReqServerChannel` -- only one of the two is ever active for a given master process. Architecture:: @@ -1449,7 +1449,7 @@ async def handle_and_route_message(self, payload): # Block non-_auth requests until this node is a committed Raft voter. if cmd != "_auth" and not _cluster_is_ready(self.opts): - log.debug("Cluster not ready yet — deferring %s request", cmd) + log.debug("Cluster not ready yet -- deferring %s request", cmd) return {"enc": "clear", "load": {"ret": False, "cluster_retry": True}} if pool_name not in self.pool_clients: @@ -1852,6 +1852,22 @@ def __init__( self._discover_event = None self._discover_token = None self._discover_candidates = {} + # Set of peer ids we have received a ``cluster/peer/discover-reply`` + # from. Populated in the discover-reply branch of + # ``handle_pool_publish`` and consumed by + # ``_pending_discover_peers`` so the periodic retry stops once each + # peer has confirmed the discover handshake. Falling back to + # ``peer_keys`` would be unsafe here -- the AES-key broadcast + # (``send_aes_key_event``) populates ``peer_keys`` independently + # of the discover protocol, so a retry keyed on ``peer_keys`` + # cancels before any ``discover-reply`` has actually landed. + self._discover_replies = set() + # Tornado ``PeriodicCallback`` driving ``_schedule_discover_retry`` + # when this master is a non-founder joining an existing cluster. + # None when inactive -- set in ``_publish_daemon`` and cleared + # once every peer has replied to the discover or the retry + # deadline expires. + self._discover_retry_pc = None # Set by service.py once the Raft node is started. self._raft_dispatcher = None @@ -1868,7 +1884,7 @@ def _start_raft_as_founding_voter(self): return # already started by a join-reply race log.info( - "No join-reply received — starting Raft as founding voter for cluster %r", + "No join-reply received -- starting Raft as founding voter for cluster %r", self.opts["cluster_id"], ) try: @@ -1906,7 +1922,7 @@ def _start_raft_as_learner(self, known_peers): ``io_loop.start()`` so the asyncio loop is already running. :param known_peers: dict ``{peer_id: pub_key_pem}`` from the - ``join-reply`` payload — the addresses of all + ``join-reply`` payload -- the addresses of all existing cluster members. """ if self._raft_service is not None: @@ -1984,7 +2000,7 @@ def _handle_multi_ring_runner_event(self, tag, data): elif tag == "cluster/runner/route_clear": svc.propose_route(data["data_type"], None) elif tag == "cluster/runner/ring_set": - # Per-ring policy commit — proposes RING_CONFIG on + # Per-ring policy commit -- proposes RING_CONFIG on # the named ring's *own* log. The ring's Node must # be the leader of its own group on this master. ring_id = data["ring_id"] @@ -2049,7 +2065,7 @@ async def _fanout_multi_ring_request(self, tag, data): cluster_aes-encrypted ``cluster/peer/multi-ring-request`` event and push it to every peer; each peer's handler re-dispatches via :meth:`_handle_multi_ring_runner_event`, but only the leader's - propose actually appends a log entry — followers log "not + propose actually appends a log entry -- followers log "not leader" and skip. This makes the runner location-independent without requiring an explicit "who's the leader" lookup. """ @@ -2083,7 +2099,7 @@ async def _run_delegate_write(self, payload): Originator side of delegate-on-miss. The local ``cluster/runner/delegate_write`` event (fired by ``EventMonitor._delegate_on_miss``) carries a fully-formed - write request — we just have to find the owner's pusher and + write request -- we just have to find the owner's pusher and send it. """ owner = payload.get("owner") @@ -2158,11 +2174,535 @@ def _handle_delegate_write(self, payload): payload.get("ring_id"), ) + async def publish_cache_write( + self, + bank, + key, + data, + expires, + originating_master, + batch_items=None, + target_master=None, + ): + """ + Fan a cache write out to every cluster peer -- or deliver it to + a single named peer. + + Originator side of default cross-master cache replication -- + wraps a ``salt.cache.Cache`` write in a cluster_aes-encrypted + ``cluster/peer/delegate-cache-write`` event and pushes it to + every peer's pool channel. Each peer's handler re-applies the + write locally via :meth:`_handle_delegate_cache_write`. The + ``originating_master`` field lets the peer drop the echoed + event if the bus bounced it back to us. + + When ``batch_items`` is a list of ``[key, data]`` pairs the + event carries a batched payload (``batch: True``, + ``items: batch_items``). The receiver's + :func:`_handle_delegate_cache_write` applies every pair under + the same loop-guard flag. ``key``/``data`` are ignored on + the batched path. + + When ``target_master`` is a peer ``pull_host`` the event is + delivered only to that peer -- used by the ring-routed branch + of :meth:`_run_delegate_cache_write` to send a single-owner + write without fanning out. ``None`` keeps the broadcast + behaviour. + + Parallel to the existing publish logic for + ``cluster/peer/delegate-write`` (ring-routed, single owner). + """ + try: + crypticle = salt.crypt.Crypticle( + self.opts, + salt.master.SMaster.secrets["cluster_aes"]["secret"].value, + ) + except Exception: # pylint: disable=broad-except + log.exception("cache-replicate: cluster_aes unavailable") + return + if batch_items is not None: + payload = { + "bank": bank, + "batch": True, + "items": batch_items, + "expires": expires, + "originating_master": originating_master, + } + label = f"batch[{len(batch_items)}]" + else: + payload = { + "bank": bank, + "key": key, + "data": data, + "expires": expires, + "originating_master": originating_master, + } + label = str(key) + event = salt.utils.event.SaltEvent.pack( + salt.utils.event.tagify("delegate-cache-write", "peer", "cluster"), + crypticle.dumps(payload), + ) + if target_master is None: + targets = list(self.pushers) + else: + targets = [p for p in self.pushers if p.pull_host == target_master] + if not targets: + log.warning( + "cache-replicate: no pusher for target_master %s; " + "dropping %s/%s (known peers: %s)", + target_master, + bank, + label, + [p.pull_host for p in self.pushers], + ) + return + for pusher in targets: + try: + await pusher.publish(event) + except Exception: # pylint: disable=broad-except + log.exception( + "cache-replicate: failed to send %s/%s to %s", + bank, + label, + pusher.pull_host, + ) + + async def publish_resource_register_write( + self, + minion_id, + resources, + originating_master, + target_master=None, + chunk_final=True, + chunk_scope=None, + chunk_index=None, + ): + """ + Dispatch a resource-registry register across the cluster. + + Originator side of resource-registry replication -- wraps the + minion-id + resources payload in a cluster_aes-encrypted + ``cluster/peer/delegate-resource-register-write`` event. + + Two modes: + + * *target_master* ``None`` (broadcast). Pushes the event to + every peer's pool channel -- today's N-of-N semantics. + * *target_master* set to a peer ``pull_host``. Pushes to + exactly that peer. Used by the ``resource_index`` ring + routing path in :func:`fire_resource_register_write` so + each peer receives only the SRNs it owns. + + The ``originating_master`` field lets the peer drop the echoed + event if the bus bounced it back to us. Parallel to + :meth:`publish_cache_write` but for the mmap-backed registry + that bypasses the :class:`salt.cache.Cache` facade. + """ + try: + crypticle = salt.crypt.Crypticle( + self.opts, + salt.master.SMaster.secrets["cluster_aes"]["secret"].value, + ) + except Exception: # pylint: disable=broad-except + log.exception("resource-replicate: cluster_aes unavailable") + return + payload = { + "minion_id": minion_id, + "resources": resources, + "originating_master": originating_master, + "chunk_final": chunk_final, + "chunk_scope": chunk_scope or "__all__", + } + if chunk_index is not None: + payload["chunk_index"] = chunk_index + event = salt.utils.event.SaltEvent.pack( + salt.utils.event.tagify( + "delegate-resource-register-write", "peer", "cluster" + ), + crypticle.dumps(payload), + ) + if target_master is None: + targets = list(self.pushers) + else: + targets = [p for p in self.pushers if p.pull_host == target_master] + if not targets: + log.warning( + "resource-replicate: no pusher for target_master %s; " + "dropping targeted register for minion %s " + "(known peers: %s)", + target_master, + minion_id, + [p.pull_host for p in self.pushers], + ) + return + for pusher in targets: + try: + await pusher.publish(event) + except Exception: # pylint: disable=broad-except + log.exception( + "resource-replicate: failed to send register for %s to %s", + minion_id, + pusher.pull_host, + ) + + def _handle_delegate_resource_register_write(self, payload): + """ + Peer-side handler for ``cluster/peer/delegate-resource-register-write``. + + Thin wrapper over + :func:`salt.cache._cluster_replicator._handle_delegate_resource_register_write`; + the free-function form is testable without instantiating this + channel class and is the single source of truth for the + peer-apply logic. + """ + import salt.cache._cluster_replicator # pylint: disable=import-outside-toplevel + + return salt.cache._cluster_replicator._handle_delegate_resource_register_write( + self.opts, payload + ) + + async def _run_delegate_resource_register_write(self, payload): + """ + Dispatch a local ``cluster/runner/delegate_resource_register_write`` + event. + + The master's ``__register_resources_sync`` fires this event + from the MWorker (which has no ring state because it forked + before any ring was created). This method runs inside the + publish daemon process where ``ring_membership`` is populated, + so it's the single place that decides between broadcast and + ring-routed fan-out and applies the local-shard write + accordingly. + + Three modes: + + * **Standalone / shared-FS**: MWorker already wrote locally + and would never fire this event -- not reached. + * **Broadcast (no ``resource_index`` ring)**: write the full + set locally, fan the event to every peer with the full set. + * **Ring-routed**: partition by owner. Apply the self-owned + share locally, fire one targeted event per remote owner + carrying only that owner's SRNs. + + Chunked-register plumbing: ``chunk_final`` + ``chunk_scope`` + flow through the local write and every peer publish so a + multi-chunk register preserves additive mid-stream puts and + defers the stale-SRN prune to the terminal chunk on every + peer. Missing fields default to ``True`` / ``"__all__"`` for + back-compat with legacy single-shot payloads. + """ + minion_id = payload.get("minion_id") + resources = payload.get("resources") or {} + originating_master = payload.get("originating_master") + chunk_final = payload.get("chunk_final", True) + chunk_scope = payload.get("chunk_scope") or "__all__" + chunk_index = payload.get("chunk_index") + import salt.cache._cluster_replicator # pylint: disable=import-outside-toplevel + import salt.utils.minions # pylint: disable=import-outside-toplevel + + local_bucket, peer_buckets, ring_active = ( + salt.cache._cluster_replicator.partition_resources_for_dispatch( + self.opts, resources + ) + ) + if local_bucket: + try: + salt.utils.minions.update_resource_index_chunk( + self.opts, + minion_id, + local_bucket, + chunk_final, + chunk_scope=chunk_scope, + chunk_index=chunk_index, + ) + except Exception: # pylint: disable=broad-except + log.exception( + "resource-replicate: local %s-mode write failed for %s", + "ring" if ring_active else "broadcast", + minion_id, + ) + if not ring_active: + # Broadcast: one event, every peer, full set. + await self.publish_resource_register_write( + minion_id, + resources, + originating_master, + chunk_final=chunk_final, + chunk_scope=chunk_scope, + chunk_index=chunk_index, + ) + return + # Ring-routed: one event per remote owner, carrying only its SRNs. + log.debug( + "resource-replicate: ring dispatch for minion=%s local_shard=%d " + "peer_shards=%s final=%s scope=%s", + minion_id, + sum(len(v) for v in local_bucket.values()), + {k: sum(len(v) for v in r.values()) for k, r in peer_buckets.items()}, + chunk_final, + chunk_scope, + ) + for owner, owner_resources in peer_buckets.items(): + await self.publish_resource_register_write( + minion_id, + owner_resources, + originating_master, + target_master=owner, + chunk_final=chunk_final, + chunk_scope=chunk_scope, + chunk_index=chunk_index, + ) + + def _handle_delegate_cache_write(self, payload): + """ + Peer-side handler for ``cluster/peer/delegate-cache-write``. + + Thin wrapper over + :func:`salt.cache._cluster_replicator._handle_delegate_cache_write`; + the free-function form is testable without instantiating this + channel class and is the single source of truth for the + peer-apply logic. + + Barrier: when a state-sync session is in flight and still + shipping a catch-up snapshot for the same top-level bank + this delta belongs to, queue the delta and apply it after + the matching ``bank_sync:`` channel eofs. Without this + barrier a late stale-snapshot chunk can overwrite a fresh + live delta ("stale wins" ordering). + """ + import salt.cache._cluster_replicator # pylint: disable=import-outside-toplevel + + bank = None + if isinstance(payload, dict): + bank = payload.get("bank") + if bank and self._cache_bank_barrier_active(bank): + self._queue_cache_bank_delta(bank, payload) + return None + return salt.cache._cluster_replicator._handle_delegate_cache_write( + self.opts, payload + ) + + def _cache_bank_barrier_active(self, bank): + """ + Return True if any registered state-sync session is still + catching up the top-level bank *bank* belongs to. + + Uses longest-prefix match on the barrier's bank name so a + compound delta bank (``grains/``) matches a top-level + barrier bank (``grains``). + """ + barriers = getattr(self, "_cache_bank_barriers", None) or {} + if not barriers: + return False + for banks in barriers.values(): + for pending in banks: + if bank == pending or bank.startswith(pending + "/"): + return True + return False + + def _queue_cache_bank_delta(self, bank, payload): + """ + Queue a live ``delegate-cache-write`` payload behind the + matching pending bank's catch-up stream. + """ + barriers = getattr(self, "_cache_bank_barriers", None) or {} + for banks in barriers.values(): + for pending, queue in banks.items(): + if bank == pending or bank.startswith(pending + "/"): + queue.append(payload) + return + + def _release_cache_bank_barrier(self, session_id, bank): + """ + Drain the queued live writes for *bank* on *session_id* and + drop the bank from the session's barrier set. + + Called from :meth:`_apply_state_sync_chunk` when a + ``bank_sync:`` channel reports eof. Applied via the + same free-function handler the live path uses so the + ``originating_master`` guard and ``_already_replicated`` + flag flow through identically. + """ + import salt.cache._cluster_replicator # pylint: disable=import-outside-toplevel + + barriers = getattr(self, "_cache_bank_barriers", None) or {} + session_banks = barriers.get(session_id) + if not session_banks: + return + queue = session_banks.pop(bank, None) or [] + if not session_banks: + barriers.pop(session_id, None) + for queued_payload in queue: + try: + salt.cache._cluster_replicator._handle_delegate_cache_write( + self.opts, queued_payload + ) + except Exception: # pylint: disable=broad-except + log.exception( + "state-sync barrier: failed to replay queued " + "delegate-cache-write for %s", + bank, + ) + if queue: + log.info( + "state-sync %s/%s barrier released; drained %d queued writes", + session_id, + bank, + len(queue), + ) + + def _drain_cache_bank_barrier(self, session_id): + """ + Drain every bank's queued writes for *session_id* (used when + on_complete fires via the watchdog rather than eof-of-all- + channels). Keeps the barrier a bounded-latency hold. + """ + barriers = getattr(self, "_cache_bank_barriers", None) or {} + session_banks = barriers.get(session_id) + if not session_banks: + return + for bank in list(session_banks.keys()): + self._release_cache_bank_barrier(session_id, bank) + + async def _run_delegate_cache_write(self, payload): + """ + Dispatch a local ``cluster/runner/delegate_cache_write`` event. + + The cache facade (``salt/cache/__init__.py:Cache.store``) fires + a single event that carries a ``write_kind`` describing the + desired cross-master routing: + + * ``"replicate"`` -> :meth:`publish_cache_write` (N-of-N). + * ``"ring"`` -> :meth:`_run_delegate_write` + (1-of-N via the existing HashRing-owner path). + + Unknown kinds are logged and dropped. + """ + write_kind = payload.get("write_kind", "replicate") + batched = bool(payload.get("batch")) + if write_kind == "replicate": + await self.publish_cache_write( + payload.get("bank"), + payload.get("key"), + payload.get("data"), + payload.get("expires"), + payload.get("originating_master"), + batch_items=payload.get("items") if batched else None, + ) + elif write_kind == "ring": + await self._run_delegate_cache_write_ring(payload, batched) + else: + log.warning("cache-replicate: unknown write_kind %r; dropping", write_kind) + + async def _run_delegate_cache_write_ring(self, payload, batched): + """ + Route a ``write_kind="ring"`` cache write to a single peer. + + Resolves the owning master for each (bank, key) via the + ``resource_index`` HashRing and publishes the delegate-write + event only to that peer. The originator already applied the + local write in :class:`salt.cache.Cache` before firing this + event, so writes whose owner is *this* master are a no-op + (we drop them); writes whose owner cannot be resolved (empty + ring, missing module) fall back to broadcast replicate so + no update is lost. + + Batched payloads are partitioned by owner: each remote owner + receives a batch containing only its own ``[key, data]`` + pairs; self-owned pairs are dropped; None-owner pairs fall + back to broadcast. + + All banks that declare ``cluster_cache_routing: : + ring`` in master opts hash their keys against the same + ``resource_index`` ring. That is the only ring populated + in the publish daemon today; if another ring is ever added + and a bank should hash against it, teach this method to + pick the ring by bank prefix. + """ + import salt.cache._cluster_replicator # pylint: disable=import-outside-toplevel + import salt.cluster # pylint: disable=import-outside-toplevel + + bank = payload.get("bank") + originating_master = payload.get("originating_master") + expires = payload.get("expires") + ring = salt.cache._cluster_replicator._get_resource_index_ring() + if ring is None or not ring.nodes(): + log.debug( + "cache-replicate: ring unavailable for %s; broadcasting", + bank, + ) + await self.publish_cache_write( + bank, + payload.get("key"), + payload.get("data"), + expires, + originating_master, + batch_items=payload.get("items") if batched else None, + ) + return + self_node = salt.cluster.node_id(self.opts) if self.opts.get("id") else None + if batched: + buckets = {} # owner -> list of [key, data] + unresolved = [] + for pair in payload.get("items") or (): + try: + key, data = pair + except (TypeError, ValueError): + continue + owner = ring.get_owner(key) + if owner is None: + unresolved.append([key, data]) + else: + buckets.setdefault(owner, []).append([key, data]) + if unresolved: + await self.publish_cache_write( + bank, + None, + None, + expires, + originating_master, + batch_items=unresolved, + ) + for owner, items in buckets.items(): + if owner == self_node: + continue + await self.publish_cache_write( + bank, + None, + None, + expires, + originating_master, + batch_items=items, + target_master=owner, + ) + return + key = payload.get("key") + owner = ring.get_owner(key) if key is not None else None + if owner is None: + await self.publish_cache_write( + bank, + key, + payload.get("data"), + expires, + originating_master, + ) + return + if owner == self_node: + return + await self.publish_cache_write( + bank, + key, + payload.get("data"), + expires, + originating_master, + target_master=owner, + ) + async def _run_shed_unowned_all(self, request_payload): """ Fan out a shed-unowned request to every peer. - Originator side of ``cluster.shed_unowned_all`` — wraps the + Originator side of ``cluster.shed_unowned_all`` -- wraps the runner-supplied payload in a ``cluster_aes``-encrypted event and publishes it to every peer's pool channel. Each peer's :meth:`handle_pool_publish` intercepts the event and runs the @@ -2230,7 +2770,7 @@ def _handle_shed_request(self, request_payload): async def _run_collect_from_peers(self, channels): """ Operator-driven *pull* of cache contents from every peer. - Mirror image of :meth:`_run_root_sync_to_peers` — instead of + Mirror image of :meth:`_run_root_sync_to_peers` -- instead of this master being the sender, this master is the receiver and asks each peer to send. @@ -2336,7 +2876,7 @@ def _record_collect_chunk(self, channel, items_installed, eof): outcome and flush. ``complete`` flips to True when every channel has seen at - least ``len(expected_peers)`` eofs — the heuristic for "every + least ``len(expected_peers)`` eofs -- the heuristic for "every peer responded for every requested channel." Operators watching the file poll for the boolean, not the per-channel counts. @@ -2369,7 +2909,7 @@ def _write_collect_sentinel(self): Persist ``self._collect_status`` to ``cachedir/cluster-collect-status.json``. - Atomic write — tmp file + rename — so an operator polling + Atomic write -- tmp file + rename -- so an operator polling the sentinel during a high-rate collect never sees a torn write. Each chunk-arrival path calls back here, so the write frequency is bounded only by chunk arrival rate. @@ -2401,9 +2941,9 @@ async def _handle_collect_request(self, requester, channels): streams the requested cache channels. Three channel shapes are accepted: - * ``keys`` / ``denied_keys`` — minion-keys banks shipped via + * ``keys`` / ``denied_keys`` -- minion-keys banks shipped via :func:`salt.cluster.state_sync.iter_keys_chunks`. - * ``bank:`` — arbitrary :class:`salt.cache.Cache` + * ``bank:`` -- arbitrary :class:`salt.cache.Cache` banks shipped via :func:`salt.cluster.state_sync.iter_bank_chunks`. Used by the multi-ring jobs migration. @@ -2439,7 +2979,7 @@ async def _handle_collect_request(self, requester, channels): # Announce the session so the requester's receiver registers # it before chunks arrive. Reuse the sync-roots-begin shape - # — the on-complete on that path is a plain teardown, which + # -- the on-complete on that path is a plain teardown, which # is what we want here too. Mark the origin so the # requester's chunk handler updates the collect-status # sentinel and not the sync_roots one. @@ -2487,14 +3027,14 @@ async def _run_root_sync_to_peers(self, channels): 1. Allocate a fresh session id. 2. Emit a ``cluster/peer/sync-roots-begin`` event to the peer so - the receiver pre-registers a state-sync session — same + the receiver pre-registers a state-sync session -- same contract as the join-reply flow, but the receiver's ``on_complete`` is a no-op (this is an ad-hoc push, not a Raft-learner bootstrap). 3. Stream the requested channels to that peer via the standard state-sync chunk format. - Errors per-peer are logged but do not stop the fan-out — a + Errors per-peer are logged but do not stop the fan-out -- a partially-online cluster can still receive the update on reachable peers. """ @@ -2508,7 +3048,7 @@ async def _run_root_sync_to_peers(self, channels): FILE_ROOTS_CHANNEL: self.opts.get("file_roots"), PILLAR_ROOTS_CHANNEL: self.opts.get("pillar_roots"), } - # Honour the channel filter — operator may want only one of the + # Honour the channel filter -- operator may want only one of the # two trees synced. active_channels = [ ch for ch in (FILE_ROOTS_CHANNEL, PILLAR_ROOTS_CHANNEL) if ch in channels @@ -2541,7 +3081,7 @@ async def _send_sync_roots_to_peer( Push one ``cluster.sync_roots`` session to a single peer. Split out of :meth:`_run_root_sync_to_peers` so the per-peer - loop variables are explicit method arguments — avoids + loop variables are explicit method arguments -- avoids cell-var-from-loop closure captures on the per-channel send. """ from salt.cluster.state_sync import ( # pylint: disable=import-outside-toplevel @@ -2628,21 +3168,30 @@ async def _send_sync_roots_channel( peer_id, ) - async def _send_state_sync_chunks(self, session_id, peer_id): + async def _send_state_sync_chunks(self, session_id, peer_id, cache_banks=None): """ Stream the four state-sync channels (keys, denied_keys, - file_roots, pillar_roots) to a freshly joined peer. + file_roots, pillar_roots) to a freshly joined peer, plus any + replicate-mode cache banks named in *cache_banks* (join-time + catch-up for peers that joined after writes already landed). Each channel runs to completion independently and emits at least one chunk (an empty chunk with ``eof=True`` if the channel has no data). All chunks are encrypted with the cluster session AES - key — the joiner has it from the join-reply we just sent. + key -- the joiner has it from the join-reply we just sent. + + :param cache_banks: Optional iterable of top-level bank names. + Each bank is streamed as a ``bank_sync:`` channel; + the joiner's ``_begin_state_sync_session`` has already + registered them on the per-session state machine. """ from salt.cluster.state_sync import ( # pylint: disable=import-outside-toplevel DENIED_CHANNEL, FILE_ROOTS_CHANNEL, KEYS_CHANNEL, PILLAR_ROOTS_CHANNEL, + bank_sync_channel, + iter_cache_bank_chunks, iter_keys_chunks, iter_root_chunks, ) @@ -2694,10 +3243,12 @@ async def send_channel(channel, chunk_iter): peer_id, ) - # Run the four channels concurrently — each finishes when it - # finishes, and a slow file_roots stream does not block keys. + # Run the four core channels concurrently -- each finishes when + # it finishes, and a slow file_roots stream does not block keys. + # Replicate-mode cache banks race alongside as extra + # ``bank_sync:`` channels. try: - await asyncio.gather( + tasks = [ send_channel(KEYS_CHANNEL, iter_keys_chunks(self.opts, KEYS_CHANNEL)), send_channel( DENIED_CHANNEL, iter_keys_chunks(self.opts, DENIED_CHANNEL) @@ -2710,7 +3261,15 @@ async def send_channel(channel, chunk_iter): PILLAR_ROOTS_CHANNEL, iter_root_chunks(self.opts.get("pillar_roots")), ), - ) + ] + for bank in cache_banks or []: + tasks.append( + send_channel( + bank_sync_channel(bank), + iter_cache_bank_chunks(self.opts, bank), + ) + ) + await asyncio.gather(*tasks) except Exception: # pylint: disable=broad-except log.exception("state-sync session %s aborted unexpectedly", session_id) finally: @@ -2731,7 +3290,9 @@ def _apply_state_sync_chunk(self, chunk): KEYS_CHANNEL, PILLAR_ROOTS_CHANNEL, bank_from_channel, + bank_from_sync_channel, install_bank_chunk, + install_bank_sync_chunk, install_keys_chunk, install_root_chunk, ) @@ -2764,21 +3325,29 @@ def _apply_state_sync_chunk(self, chunk): elif channel == PILLAR_ROOTS_CHANNEL: installed = install_root_chunk(self.opts.get("pillar_roots"), items) else: - # ``bank:`` channels carry arbitrary cache - # entries — used by ``cluster.collect_from_peers`` - # for caches outside the four join-time channels. - bank = bank_from_channel(channel) - if bank: - installed = install_bank_chunk(self.opts, bank, items) + # ``bank_sync:`` channels are the join-time + # catch-up stream -- apply with + # ``_already_replicated=True`` so the receiver does + # NOT re-broadcast. ``bank:`` channels are + # operator-triggered collects (plain store). + sync_bank = bank_from_sync_channel(channel) + if sync_bank: + installed = install_bank_sync_chunk(self.opts, sync_bank, items) + if eof: + self._release_cache_bank_barrier(session_id, sync_bank) else: - log.warning( - "state-sync chunk for unknown channel %r " - "(session=%s seq=%s)", - channel, - session_id, - seq, - ) - return + bank = bank_from_channel(channel) + if bank: + installed = install_bank_chunk(self.opts, bank, items) + else: + log.warning( + "state-sync chunk for unknown channel %r " + "(session=%s seq=%s)", + channel, + session_id, + seq, + ) + return except Exception: # pylint: disable=broad-except log.exception( "state-sync %s/%s seq=%s install failed", @@ -2796,30 +3365,46 @@ def _apply_state_sync_chunk(self, chunk): " (eof)" if eof else "", ) # Collect-origin sessions update the operator-visible - # status sentinel — bumps a counter the operator polls + # status sentinel -- bumps a counter the operator polls # while waiting for the runner's fan-out to finish. if getattr(session, "origin", "sync_roots") == "collect": self._record_collect_chunk(channel, installed, eof) session.record_chunk(channel, seq, eof, installed) - def _begin_state_sync_session(self, session_id, known_peers, discover_event): + def _begin_state_sync_session( + self, session_id, known_peers, discover_event, cache_banks=None + ): """ Register a state-sync session and arm its watchdog timer. Called from the join-reply handler once we know the responder is running with ``cluster_isolated_filesystem=True`` and intends to - push the four chunked channels at us. The session's - ``on_complete`` callback fires - :meth:`_start_raft_as_learner` exactly once, either when all four + push the chunked channels at us. The session's + ``on_complete`` callback fires :meth:`_start_raft_as_learner` + exactly once, either when all core AND cache-bank catch-up channels report eof or when the deadline expires. + + :param cache_banks: Optional iterable of top-level bank names + the responder named in the join-reply. Each bank gets a + ``bank_sync:`` channel registered on the session and + added to the per-session barrier set so live + ``delegate-cache-write`` deltas for those banks are + queued until the matching catch-up channel eofs. """ from salt.cluster.state_sync import ( # pylint: disable=import-outside-toplevel + ALL_CHANNELS, DEFAULT_RECEIVE_TIMEOUT, StateSyncSession, + bank_sync_channel, ) if not hasattr(self, "_state_sync_sessions"): self._state_sync_sessions = {} + if not hasattr(self, "_cache_bank_barriers"): + # Per-session {bank: [queued_payload, ...]} mapping used + # by ``_handle_delegate_cache_write`` to queue live writes + # until the matching ``bank_sync:`` channel eofs. + self._cache_bank_barriers = {} if session_id in self._state_sync_sessions: log.warning( @@ -2828,6 +3413,9 @@ def _begin_state_sync_session(self, session_id, known_peers, discover_event): ) return + banks = list(cache_banks or []) + channels = list(ALL_CHANNELS) + [bank_sync_channel(b) for b in banks] + completed_holder = {"done": False} def _on_complete(): @@ -2849,14 +3437,19 @@ def _on_complete(): handle.cancel() except Exception: # pylint: disable=broad-except pass - # Drop the session from the registry — keep memory bounded. + # Drain any still-queued barriered writes (deadline-forced + # completion may leave queued items behind). + self._drain_cache_bank_barrier(session_id) + # Drop the session from the registry -- keep memory bounded. self._state_sync_sessions.pop(session_id, None) - session = StateSyncSession(session_id, _on_complete) + session = StateSyncSession(session_id, _on_complete, channels=channels) # Stash the watchdog handle on the session so on_complete can # cancel it; created below. session.watchdog_handle = None self._state_sync_sessions[session_id] = session + if banks: + self._cache_bank_barriers[session_id] = {bank: [] for bank in banks} try: loop = asyncio.get_event_loop() @@ -2864,7 +3457,7 @@ def _on_complete(): DEFAULT_RECEIVE_TIMEOUT, session.force_complete ) except RuntimeError: - # No running event loop in this context (defensive — the + # No running event loop in this context (defensive -- the # join-reply handler runs inside ``_publish_daemon``'s loop, # so this branch should not execute in production). log.warning( @@ -2880,10 +3473,10 @@ def _begin_root_sync_session(self, session_id, channels, origin="sync_roots"): *origin* distinguishes between session shapes that share the same wire format: - * ``"sync_roots"`` (default) — push from + * ``"sync_roots"`` (default) -- push from ``cluster.sync_roots``; the on-complete only tears down the registry entry. - * ``"collect"`` — pull driven by + * ``"collect"`` -- pull driven by ``cluster.collect_from_peers``; the chunk handler also updates ``cachedir/cluster-collect-status.json`` so an operator can confirm completion without tailing logs. @@ -2933,7 +3526,7 @@ def _on_complete(): self._state_sync_sessions.pop(session_id, None) # The StateSyncSession state machine tracks per-channel eof. We - # tell it about only the channels we expect — when each fires + # tell it about only the channels we expect -- when each fires # eof, the session triggers on_complete. session = StateSyncSession(session_id, _on_complete, channels=channels) session.watchdog_handle = None @@ -2958,7 +3551,7 @@ def _join_sentinel_path(self): The filename is namespaced by the master's node-id so that deployments which share ``cachedir`` between cluster members (and the cluster integration tests, which point every master at the same - ``cluster_cache_path``) keep distinct sentinels — without that, the + ``cluster_cache_path``) keep distinct sentinels -- without that, the first master to join writes ``.cluster_joined`` and every later master sees it on startup, takes the "rejoining" path, and skips the deterministic founding-voter election. @@ -2981,7 +3574,7 @@ def _mark_joined_cluster(self): except OSError as exc: log.warning("Could not write cluster join sentinel %s: %s", sentinel, exc) - def discover_peers(self): + def discover_peers(self, peers=None): """ Send a ``cluster/peer/discover`` event to each configured peer. @@ -2990,6 +3583,14 @@ def discover_peers(self): Existing peers will reply with ``cluster/peer/discover-reply``, which triggers the full join handshake and eventually ``cluster/peer/join-reply`` received by ``handle_pool_publish``. + + :param peers: Optional iterable of peer ids to send discover to. When + ``None`` (the default) every entry in + ``self.cluster_peers`` is contacted. The retry path in + :meth:`_schedule_discover_retry` passes the subset of + peers for which no reply has been seen yet, so we do + not re-spam peers that have already completed the + handshake. """ path = self.master_key.master_pub_path with salt.utils.files.fopen(path, "r") as fp: @@ -2997,7 +3598,9 @@ def discover_peers(self): self._discover_token = self.gen_token() - for peer in self.cluster_peers: + target_peers = list(self.cluster_peers if peers is None else peers) + + for peer in target_peers: log.info("Sending cluster discover to peer %s", peer) tosign = salt.payload.package( { @@ -3023,6 +3626,115 @@ def discover_peers(self): if not success: log.error("Unable to send cluster discover event to %s", peer) + def _pending_discover_peers(self): + """ + Return the list of configured cluster peers that have not yet + answered this master's ``cluster/peer/discover`` with a signed + ``cluster/peer/discover-reply``. + + Used by :meth:`_schedule_discover_retry` to drive a periodic + re-send of ``cluster/peer/discover`` for peers that missed the + initial fire-and-forget attempt. The cancellation set + (``self._discover_replies``) is populated only after the + discover-reply branch has verified the signature and cluster-pub + fingerprint, so a peer that is merely reachable (``peer_keys`` + populated by the independent AES-key broadcast) does not count + as done. + """ + pending = [] + replies = getattr(self, "_discover_replies", set()) or set() + for peer in self.cluster_peers: + # ``cluster_peers`` entries may carry an optional ``:port`` + # suffix; the peer-id used in the discover-reply handler is + # the bare host (``payload["peer_id"]``). Match on either + # form so operators who list peers as ``host:port`` still + # see a correct pending set. + host = peer.rsplit(":", 1)[0] if ":" in peer else peer + if host not in replies and peer not in replies: + pending.append(peer) + return pending + + def _schedule_discover_retry(self): + """ + Install a periodic retry of :meth:`discover_peers` for peers that + did not reply to the initial one-shot attempt from + ``salt.master.Master.start``. + + The initial ``discover_peers()`` call is fire-and-forget: if the + peer's pool-puller TCP listener is not yet bound when the discover + event is forwarded, the push raises ``StreamClosedError`` and the + event is silently lost. Without a retry the joining master waits + forever on ``_discover_event`` and the whole cluster bootstrap + stalls. + + The retry cancels itself when either (a) every peer in + ``self.cluster_peers`` has registered its AES key (i.e. the + handshake completed), or (b) ``cluster_join_discover_retry_timeout`` + seconds have elapsed. The retry interval is controlled by + ``cluster_join_discover_retry_interval`` (default 2 s) and the + timeout defaults to 60 s. + + Called from ``_publish_daemon`` in the non-founder branch where + the tornado io_loop is running. Safe to call more than once; a + previously-scheduled retry is cancelled first. + """ + if getattr(self, "_discover_retry_pc", None) is not None: + try: + self._discover_retry_pc.stop() + except Exception: # pylint: disable=broad-except + pass + self._discover_retry_pc = None + + interval_s = float(self.opts.get("cluster_join_discover_retry_interval", 2)) + timeout_s = float(self.opts.get("cluster_join_discover_retry_timeout", 60)) + if interval_s <= 0 or timeout_s <= 0: + return + if not self.cluster_peers: + return + + deadline = self.io_loop.time() + timeout_s + interval_ms = max(1, int(interval_s * 1000)) + + def _retry(): + try: + pending = self._pending_discover_peers() + if not pending: + log.info("Cluster discover complete -- stopping periodic retry") + pc = self._discover_retry_pc + self._discover_retry_pc = None + if pc is not None: + try: + pc.stop() + except Exception: # pylint: disable=broad-except + pass + return + if self.io_loop.time() >= deadline: + log.warning( + "cluster_join_discover_retry_timeout (%ss) exhausted " + "with pending peers %r; giving up on discover retry", + timeout_s, + pending, + ) + pc = self._discover_retry_pc + self._discover_retry_pc = None + if pc is not None: + try: + pc.stop() + except Exception: # pylint: disable=broad-except + pass + return + log.info( + "Retrying cluster/peer/discover for pending peers %r", + pending, + ) + self.discover_peers(peers=pending) + except Exception: # pylint: disable=broad-except + log.exception("Error in cluster discover retry callback") + + pc = tornado.ioloop.PeriodicCallback(_retry, interval_ms) + self._discover_retry_pc = pc + pc.start() + def send_aes_key_event(self): log.debug("Sending AES key event") # ``cluster_peers`` is documented to hold bare master names so the @@ -3166,7 +3878,7 @@ def _publish_daemon(self, **kwargs): # Start the Raft node when this master is part of a cluster. # A node without a cluster private key hasn't completed the join - # handshake yet — defer Raft startup to _start_raft_as_learner, which + # handshake yet -- defer Raft startup to _start_raft_as_learner, which # is called from handle_pool_publish when join-reply arrives. self._raft_service = None if self.opts.get("cluster_id") and self.opts.get("cluster_peers"): @@ -3203,7 +3915,7 @@ def _publish_daemon(self, **kwargs): # masters' timers expire before they can exchange # join-replies and each one bootstraps its own single-member # cluster, leaving the cluster with multiple disjoint - # leaders or — when join-replies land first — zero voters. + # leaders or -- when join-replies land first -- zero voters. node_id = salt.cluster.node_id(self.opts) bootstrap_pool = sorted({node_id, *self.opts.get("cluster_peers", [])}) aio_loop_deferred = salt.utils.asynchronous.aioloop(self.io_loop) @@ -3217,7 +3929,7 @@ def _publish_daemon(self, **kwargs): # inbound join-reply can race this start-up. Still # delay by ``cluster_join_timeout`` before starting # Raft so peer masters have time to bring up their - # cluster pool pullers — the very first ``pre-vote`` + # cluster pool pullers -- the very first ``pre-vote`` # the founder fires must reach at least one peer to # form quorum, otherwise the node never re-arms its # election timer. @@ -3227,9 +3939,18 @@ def _publish_daemon(self, **kwargs): ) else: log.info( - "New node joining cluster %r — waiting for join-reply to start Raft as learner", + "New node joining cluster %r -- waiting for join-reply to start Raft as learner", self.opts["cluster_id"], ) + # The initial fire-and-forget ``discover_peers`` in + # ``salt.master.Master.start`` can be lost silently if + # the peer's pool-puller TCP listener is not yet bound + # when the discover event is forwarded. Install a + # periodic retry that re-sends discover to peers whose + # AES key has not yet arrived. See + # ``_schedule_discover_retry`` for the cancellation + # conditions. + self._schedule_discover_retry() # run forever try: self.io_loop.start() @@ -3256,7 +3977,7 @@ def _signal_cluster_ready(self): entry = salt.master.SMaster.secrets.get("cluster_ready") if entry is not None: - log.info("MasterPubServerChannel: cluster ready — opening request gate") + log.info("MasterPubServerChannel: cluster ready -- opening request gate") entry["event"].set() salt.cluster.healthchecks.mark_cluster_ready(self.opts) @@ -3314,7 +4035,7 @@ def _add_pusher(self, pusher): (publish_payload's cluster-event broadcast, sync_roots, collect_from_peers, shed_unowned_all, delegate-on-miss) iterates over. A duplicate entry causes every event to ship - twice — wasted bandwidth and, more dangerously, non-atomic + twice -- wasted bandwidth and, more dangerously, non-atomic sentinel writes that interleave under concurrent arrivals on the peer side. Multiple code paths add pushers (static config, join-reply, discover-reply, late-joiner @@ -3358,7 +4079,8 @@ async def handle_pool_publish(self, payload): "Raft RPC received but dispatcher not initialised: %s", tag ) return - log.debug("Incomming from peer %s %r", tag, data) + if log.isEnabledFor(5): # TRACE level + log.log(5, "Incoming from peer %s %r", tag, data) if tag.startswith("cluster/peer/state-sync-chunk"): # Encrypted with the shared cluster_aes the joiner just # installed in the matching join-reply. Each chunk @@ -3419,6 +4141,88 @@ async def handle_pool_publish(self, payload): loop = asyncio.get_event_loop() loop.run_in_executor(None, self._handle_delegate_write, payload) return + if tag.startswith("cluster/peer/delegate-cache-write"): + # Cross-master cache replication arrival: another + # master's ``salt.cache.Cache.store`` call fanned + # out a write. Decrypt (cluster_aes) and apply the + # write locally with the ``_already_replicated`` + # flag set so the peer-side ``Cache.store`` does + # NOT re-broadcast. + try: + crypticle = salt.crypt.Crypticle( + self.opts, + salt.master.SMaster.secrets["cluster_aes"]["secret"].value, + ) + payload = crypticle.loads(data) + except Exception: # pylint: disable=broad-except + log.exception("Failed to decrypt delegate-cache-write") + return + loop = asyncio.get_event_loop() + loop.run_in_executor(None, self._handle_delegate_cache_write, payload) + return + if tag.startswith("cluster/peer/delegate-resource-register-write"): + # Cross-master resource-registry replication arrival: + # another master's ``__register_resources_sync`` fanned + # out a register. Decrypt (cluster_aes) and apply the + # register locally; the free-function handler drops + # events whose ``originating_master`` equals our own + # ``opts["id"]`` so bus echo cannot loop. + # + # Dispatch per-minion to a dedicated single-worker + # executor (not the default ``None`` executor) so + # chunks for one minion's registration window arrive + # at :func:`_handle_delegate_resource_register_write` + # in submission order. Different minions each get + # their own executor and run in parallel. + try: + crypticle = salt.crypt.Crypticle( + self.opts, + salt.master.SMaster.secrets["cluster_aes"]["secret"].value, + ) + payload = crypticle.loads(data) + except Exception: # pylint: disable=broad-except + log.exception("Failed to decrypt delegate-resource-register-write") + return + # Import the submodule WITHOUT binding the top-level + # ``salt`` name as a function-scope local. A plain + # ``import salt.cache._cluster_replicator`` statement + # here shadows the module-level ``import salt.*`` for + # the entire ``handle_pool_publish`` body, making every + # earlier ``salt.utils.event...`` / + # ``salt.cluster.consensus...`` reference raise + # ``UnboundLocalError: local variable 'salt' referenced + # before assignment`` -- which drops every incoming + # cluster/peer event and prevents the Raft group from + # forming. ``importlib.import_module`` loads the + # module into ``sys.modules`` without introducing a + # local name, so references like ``salt.cache._cluster_replicator`` + # keep resolving through the module-level ``salt``. + import importlib # pylint: disable=import-outside-toplevel + + importlib.import_module("salt.cache._cluster_replicator") + + minion_id = payload.get("minion_id") + if not minion_id: + # Malformed payload: the free-function handler + # will log + drop. Use the default executor + # since we have no minion-id to key on. + loop = asyncio.get_event_loop() + loop.run_in_executor( + None, + self._handle_delegate_resource_register_write, + payload, + ) + return + executor = salt.cache._cluster_replicator._get_peer_apply_executor( + minion_id + ) + loop = asyncio.get_event_loop() + loop.run_in_executor( + executor, + self._handle_delegate_resource_register_write, + payload, + ) + return if tag.startswith("cluster/peer/shed-request"): # Operator-driven shed fan-out from a peer that ran # ``cluster.shed_unowned_all``. Decrypt the payload @@ -3546,7 +4350,7 @@ async def handle_pool_publish(self, payload): # encrypted to our master pub here so a joiner without access # to a shared ``cluster_pki_dir/.aes`` can adopt the cluster # session. ``cluster.pem`` (cluster RSA private) is still - # expected to be present locally — wire delivery for it is + # expected to be present locally -- wire delivery for it is # tracked separately. token = self._discover_token or "" if isinstance(token, str): @@ -3675,9 +4479,13 @@ async def handle_pool_publish(self, payload): # (or the per-session deadline elapses). known_peers = {p: inner["peers"][p] for p in inner.get("peers", {})} state_sync_session = inner.get("state_sync_session") + state_sync_cache_banks = inner.get("state_sync_cache_banks") or [] if state_sync_session and self.opts.get("cluster_isolated_filesystem"): self._begin_state_sync_session( - state_sync_session, known_peers, event + state_sync_session, + known_peers, + event, + cache_banks=state_sync_cache_banks, ) else: # Either the responder isn't running with isolated-FS @@ -3775,7 +4583,7 @@ async def handle_pool_publish(self, payload): # Add the joining peer to our own Raft state as a learner. # The join-notify broadcast below tells *other* peers about # the new node, but the receiver of the join request never - # sees its own broadcast — without this call the leader + # sees its own broadcast -- without this call the leader # learned about its peers via cluster_peers but never began # replicating to a freshly joined master, so promotion to # voter (and the joiner's gate opening) stalled. @@ -3893,13 +4701,36 @@ async def handle_pool_publish(self, payload): # a Raft learner; per-channel chunking lets each stream # progress independently and isolates failures. state_sync_session_id = None + state_sync_cache_banks = [] if self.opts.get("cluster_isolated_filesystem"): from salt.cluster.state_sync import ( # pylint: disable=import-outside-toplevel + enumerate_replicate_banks, new_session_id, ) state_sync_session_id = new_session_id() inner_payload["state_sync_session"] = state_sync_session_id + # Catch-up list: every replicate-mode bank this + # responder currently holds. The joiner uses this + # to pre-register the extra ``bank_sync:`` + # channels on its StateSyncSession AND to gate + # live ``delegate-cache-write`` arrivals behind + # the catch-up stream for the same bank (barrier + # ordering). + if self.opts.get("cluster_state_sync_cache_banks"): + try: + state_sync_cache_banks = enumerate_replicate_banks( + self.opts + ) + except Exception: # pylint: disable=broad-except + log.exception( + "state-sync: enumerate_replicate_banks failed; " + "shipping join-reply without cache-bank catch-up" + ) + state_sync_cache_banks = [] + inner_payload["state_sync_cache_banks"] = list( + state_sync_cache_banks + ) tosign = salt.payload.package(inner_payload) sig = salt.crypt.PrivateKeyString(self.private_key()).sign( tosign, algorithm=self.opts["publish_signing_algorithm"] @@ -3919,15 +4750,30 @@ async def handle_pool_publish(self, payload): if state_sync_session_id is not None: asyncio.get_event_loop().create_task( self._send_state_sync_chunks( - state_sync_session_id, payload["peer_id"] + state_sync_session_id, + payload["peer_id"], + cache_banks=state_sync_cache_banks, ) ) elif tag.startswith("cluster/peer/discover-reply"): payload = salt.payload.loads(data["payload"]) - if not cluster_pub_matches_fingerprint( - self.opts, payload["cluster_pub"] - ): + received_cluster_pub = payload.get("cluster_pub") + if not received_cluster_pub: + # The peer hasn't installed its cluster.pub yet -- + # it hit the discover branch before its own join-reply + # from the founder landed. Ignore; a future discover + # round will pick up the key. See Team Alpha + # diagnosis report ``cache-replicate-alpha-diagnosis.md``. + log.debug( + "cluster/peer/discover-reply from %s arrived " + "without cluster_pub (peer has not installed " + "cluster.pub yet); ignoring", + payload.get("peer_id"), + ) + return + + if not cluster_pub_matches_fingerprint(self.opts, received_cluster_pub): log.warning( "cluster_pub fingerprint mismatch in discover-reply " "from %s; rejecting", @@ -3935,7 +4781,7 @@ async def handle_pool_publish(self, payload): ) return - cluster_pub = salt.crypt.PublicKeyString(payload["cluster_pub"]) + cluster_pub = salt.crypt.PublicKeyString(received_cluster_pub) if not cluster_pub.verify( data["payload"], data["sig"], @@ -3952,6 +4798,13 @@ async def handle_pool_publish(self, payload): # return log.info("Cluster discover reply from %s", payload["peer_id"]) + # Record the peer as having answered the discover so the + # periodic retry in ``_schedule_discover_retry`` stops + # re-sending to it. See ``_pending_discover_peers`` for + # the matching cancellation check. + if not hasattr(self, "_discover_replies"): + self._discover_replies = set() + self._discover_replies.add(payload["peer_id"]) key = salt.crypt.PublicKeyString(payload["pub"]) self._discover_token = self.gen_token() tosign = salt.payload.package( @@ -4026,6 +4879,20 @@ async def handle_pool_publish(self, payload): log.warning("Invalid signature of cluster discover payload") return log.info("Cluster discovery from %s", payload["peer_id"]) + cluster_pub_pem = self.cluster_public_key() + if not cluster_pub_pem: + # We don't have cluster.pub on disk yet (our own + # join-reply from the founder hasn't landed). Don't + # ship a discover-reply with cluster_pub=None -- the + # peer's handler used to crash on that; even with the + # guard it would just reject us. Retry on the next + # discover round. + log.debug( + "Deferring discover-reply to %s: cluster.pub " + "not yet installed locally", + payload.get("peer_id"), + ) + return token = self.gen_token() # Store this peer as a candidate. # XXX Add timestamp so we can clean up old candidates @@ -4036,7 +4903,7 @@ async def handle_pool_publish(self, payload): "token": token, "peer_id": self.opts["id"], "pub": self.public_key(), - "cluster_pub": self.cluster_public_key(), + "cluster_pub": cluster_pub_pem, } ) key = salt.crypt.PrivateKeyString(self.cluster_key()) @@ -4188,7 +5055,7 @@ def _decode_data(): # peer; each peer's daemon runs the same shed logic and # writes a per-master sentinel. The originator runner # subprocess (which fired this event) also ran its own - # local shed inline — no need to repeat that here. + # local shed inline -- no need to repeat that here. asyncio.create_task(self._run_shed_unowned_all(_decode_data())) return if tag == "cluster/runner/delegate_write": @@ -4198,10 +5065,30 @@ def _decode_data(): # cluster_aes-encrypted ``cluster/peer/delegate-write`` # event. In the standard cluster topology bus # replication already delivered the original event to - # the owner — this delegate is a safety net for + # the owner -- this delegate is a safety net for # asymmetric topologies (or a guard against bus drops). asyncio.create_task(self._run_delegate_write(_decode_data())) return + if tag == "cluster/runner/delegate_cache_write": + # Default cross-master cache replication: the + # ``salt.cache.Cache`` facade fired this event after a + # local ``store()`` / ``store_many()``. Dispatch via + # ``_run_delegate_cache_write`` which picks between + # N-of-N replicate (encrypted peer-publish) and 1-of-N + # ring delegation based on the payload's ``write_kind``. + asyncio.create_task(self._run_delegate_cache_write(_decode_data())) + return + if tag == "cluster/runner/delegate_resource_register_write": + # Cross-master resource-registry replication: the + # master's ``__register_resources_sync`` fired this event + # after writing to the mmap-backed registry. The payload + # carries ``{minion_id, resources, originating_master}`` + # and is fanned out N-of-N to every peer master via an + # encrypted ``cluster/peer/delegate-resource-register-write``. + asyncio.create_task( + self._run_delegate_resource_register_write(_decode_data()) + ) + return if tag in ( "cluster/runner/ring_create", "cluster/runner/ring_destroy", @@ -4215,7 +5102,7 @@ def _decode_data(): # warning if we're not the leader) and *also* fan out a # cluster_aes-encrypted peer event so whichever master is # currently the leader picks it up. Followers that - # receive the fan-out log "not leader" and skip — no + # receive the fan-out log "not leader" and skip -- no # double-commit because the leader is unique. data = _decode_data() self._handle_multi_ring_runner_event(tag, data) @@ -4230,7 +5117,7 @@ def _decode_data(): ) ] for pusher in self.pushers: - log.info("Publish event to peer %s:%s", pusher.pull_host, pusher.pull_port) + log.debug("Publish event to peer %s:%s", pusher.pull_host, pusher.pull_port) if tag.startswith("cluster/peer"): # log.info("Send %s %r", tag, load) tasks.append( diff --git a/salt/cli/caller.py b/salt/cli/caller.py index ae83b15ab874..27d2d23fd4b8 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -322,8 +322,11 @@ def _call_with_resources(self): tgt_type = self.opts.get("resources_tgt_type", "glob") # Resolve resources via the inherited helper (set up on MinionBase - # so SMinion sees it). The helper consults the minion's pillar - # (``opts["resources"]``) and per-resource grains cache. + # so SMinion sees it). The helper streams per-type IDs from the + # opts-driven inventory API (``resource_types`` + each type's + # ``discover(opts)`` via ``salt.utils.resources.enabled_resource_types`` + # and ``Minion._iter_resource_ids``) and renders per-resource + # grains on demand. load = {"fun": fun, "tgt": tgt, "tgt_type": tgt_type} try: resource_targets = self.minion._resolve_resource_targets(load) diff --git a/salt/client/__init__.py b/salt/client/__init__.py index 0d8d7160ac2c..249b682dfb1d 100644 --- a/salt/client/__init__.py +++ b/salt/client/__init__.py @@ -93,36 +93,6 @@ def _resource_ids_from_minion_grains_cache(opts, minion_id): ) -def _resource_ids_from_minion_pillar_cache(opts, minion_id): - """ - Return bare resource IDs from the minion's cached pillar subtree under - :func:`~salt.utils.resources.resource_pillar_key` (default ``resources``). - - ``salt_resources`` is often absent from grains even when pillar (and thus - ``resource_ids``) was synced to the master — this path closes that gap. - """ - if not opts.get("minion_data_cache"): - return [] - try: - cache = salt.cache.factory(opts) - if not cache.contains("pillar", minion_id): - return [] - pillar = cache.fetch("pillar", minion_id) or {} - except Exception as exc: # pylint: disable=broad-except - log.debug( - "Pillar cache read for minion %s failed while expanding missing returns: %s", - minion_id, - exc, - exc_info=True, - ) - return [] - key = salt.utils.resources.resource_pillar_key(opts) - subtree = pillar.get(key) - if not isinstance(subtree, dict): - return [] - return salt.utils.resources.bare_resource_ids_from_decl(subtree) - - def _job_ret_display_id(data): """ Key for one job return in CLI / nested job-return events. @@ -147,8 +117,8 @@ def _iter_failed_missing_returns(opts, found, missing_root_ids): Resource IDs are taken from the master's resource registry when present, then merged with IDs from :conf_master:`minion_data_cache` **grains** - (``salt_resources``) and **pillar** (``resources`` / ``resource_pillar_key``), - so offline minions still expand to their last-known resource rows. + (``salt_resources``) so offline minions still expand to their last-known + resource rows. """ ck = salt.utils.minions.CkMinions(opts) reported = set() @@ -199,11 +169,6 @@ def _emit_missing_for_minion(mid): continue rid_order.append(rid) seen_rid.add(rid) - for rid in _resource_ids_from_minion_pillar_cache(opts, mid): - if rid in seen_rid: - continue - rid_order.append(rid) - seen_rid.add(rid) for rid in rid_order: if rid in reported or rid in found: continue diff --git a/salt/cluster/state_sync.py b/salt/cluster/state_sync.py index 874916c83116..c00f3b98b0eb 100644 --- a/salt/cluster/state_sync.py +++ b/salt/cluster/state_sync.py @@ -80,6 +80,15 @@ # join-time channels above. BANK_CHANNEL_PREFIX = "bank:" +# Channel prefix for the join-time catch-up of replicate-mode cache +# banks. Distinct from ``bank:`` so the receiver's install path can +# distinguish an operator-triggered collect pull (apply with the normal +# ``cache.store`` -- triggers replication) from a join-time catch-up +# (apply with ``_already_replicated=True`` so the chunks the sender +# already put on the wire don't trigger another round-trip). See +# :func:`install_bank_sync_chunk`. +BANK_SYNC_CHANNEL_PREFIX = "bank_sync:" + def bank_channel(bank): """Return the wire channel name for *bank*.""" @@ -93,6 +102,18 @@ def bank_from_channel(channel): return channel[len(BANK_CHANNEL_PREFIX) :] +def bank_sync_channel(bank): + """Return the join-time catch-up wire channel name for *bank*.""" + return f"{BANK_SYNC_CHANNEL_PREFIX}{bank}" + + +def bank_from_sync_channel(channel): + """Return the bank name a ``bank_sync:`` channel was made from, or None.""" + if not channel or not channel.startswith(BANK_SYNC_CHANNEL_PREFIX): + return None + return channel[len(BANK_SYNC_CHANNEL_PREFIX) :] + + ALL_CHANNELS = ( KEYS_CHANNEL, DENIED_CHANNEL, @@ -110,10 +131,23 @@ def bank_from_channel(channel): # enough that a typical SLS tree fits in a handful of chunks. DEFAULT_ROOTS_CHUNK_BYTES = 1 * 1024 * 1024 +# Default per-chunk byte budget for a cache bank sync stream. Banks +# like ``resource_grains`` can hold tens of MB on a production master; +# 2 MB per chunk keeps each state-sync-chunk event well under the +# transport's single-frame limit while shipping a 70 MB bank in ~35 +# chunks rather than one message. +DEFAULT_BANK_SYNC_CHUNK_BYTES = 2 * 1024 * 1024 + # Default deadline (seconds) the joiner waits for all four channels to # eof before falling back to event-driven replication. DEFAULT_RECEIVE_TIMEOUT = 30 +# Internal top-level banks that MUST NOT be shipped by the join-time +# cache-bank sync: ``jobs`` is ring-routed (shipping would duplicate the +# owner's authoritative copy), the two ``cluster/`` subtrees own their +# own replication transport (Raft + state-sync-chunks). +BANK_SYNC_SKIP_PREFIXES = ("jobs", "cluster/consensus", "cluster/state_sync") + def new_session_id(): """Return a fresh session id (URL-safe, no fixed length).""" @@ -219,6 +253,139 @@ def iter_root_chunks(roots_map, byte_budget=DEFAULT_ROOTS_CHUNK_BYTES): yield chunk +def enumerate_replicate_banks(opts): + """ + Return the sorted list of top-level cache banks on this master + whose cross-master routing resolves to ``"replicate"``. + + Enumerates by walking the on-disk cachedir (one subdirectory per + top-level bank). Driver-independent for the localfs / mmap_key + paths that all isolated-FS deployments use today. + + Banks whose names start with any entry in + :data:`BANK_SYNC_SKIP_PREFIXES` (``jobs``, ``cluster/consensus``, + ``cluster/state_sync``) are always excluded even if the routing + map accidentally lists them as ``replicate``. The ``cluster`` + subtree is skipped wholesale because its subtrees own their own + replication transport. + """ + import os # pylint: disable=import-outside-toplevel + + import salt.cache # pylint: disable=import-outside-toplevel + import salt.config # pylint: disable=import-outside-toplevel + + cachedir = opts.get("cachedir") + if not cachedir or not os.path.isdir(cachedir): + return [] + + routing = salt.config.get_cluster_cache_routing(opts) + banks = [] + for entry in sorted(os.listdir(cachedir)): + if entry.startswith("."): + continue + if entry == "cluster": + continue + full = os.path.join(cachedir, entry) + if not os.path.isdir(full): + continue + if any( + entry == pref or entry.startswith(pref + "/") + for pref in BANK_SYNC_SKIP_PREFIXES + ): + continue + mode = salt.cache._resolve_bank_routing(routing, entry) + if mode == "replicate": + banks.append(entry) + return banks + + +def iter_cache_bank_chunks(opts, bank, byte_budget=DEFAULT_BANK_SYNC_CHUNK_BYTES): + """ + Yield ``items`` lists for a :class:`salt.cache.Cache` bank, chunked + by cumulative serialized-value byte budget. + + Each item is ``{"key": str, "value": any}``. A bank with no + entries still yields one empty list so the receiver's eof flag + fires uniformly. + + Chunking approximates bytes by ``len(repr(value))`` -- cheap and + good enough for ordering-of-magnitude bounding; the real wire + size is paid by the Crypticle+msgpack path that encrypts each + enclosing chunk payload. + """ + cache_driver = opts.get("cache") or opts.get("keys.cache_driver") + cache = salt.cache.Cache(opts, driver=cache_driver) + + pairs = [] + try: + dump = cache.list_all(bank, include_data=True) + pairs = list((dump or {}).items()) + except (AttributeError, salt.exceptions.SaltCacheError): + try: + for key in cache.list(bank): + value = cache.fetch(bank, key) + pairs.append((key, value)) + except salt.exceptions.SaltCacheError: + pairs = [] + + if not pairs: + yield [] + return + + chunk = [] + chunk_bytes = 0 + for key, value in pairs: + try: + entry_bytes = len(repr(value)) + len(key) + except Exception: # pylint: disable=broad-except + entry_bytes = 1024 + if chunk and chunk_bytes + entry_bytes > byte_budget: + yield chunk + chunk = [] + chunk_bytes = 0 + chunk.append({"key": key, "value": value}) + chunk_bytes += entry_bytes + if chunk: + yield chunk + + +def install_bank_sync_chunk(opts, bank, items): + """ + Apply one join-time cache-bank catch-up chunk locally. + + Writes via ``cache.store(..., _already_replicated=True)`` so the + receiver does NOT re-broadcast the state-sync payload -- the + sender already put it on the wire exactly once. Returns the + number of entries written. + + Mirrors :func:`install_bank_chunk` but with the replication-off + flag set. The two installers stay separate so a future tweak to + the catch-up path (quorum ack, resume offset, ...) doesn't leak + into the operator-triggered collect pull. + """ + if not bank: + raise ValueError("install_bank_sync_chunk: bank is required") + cache_driver = opts.get("cache") or opts.get("keys.cache_driver") + cache = salt.cache.Cache(opts, driver=cache_driver) + written = 0 + for entry in items or []: + if not isinstance(entry, dict): + continue + key = entry.get("key") + if key is None: + continue + try: + cache.store(bank, key, entry.get("value"), _already_replicated=True) + written += 1 + except Exception: # pylint: disable=broad-except + log.exception( + "state-sync: failed to install bank-sync entry %s/%s", + bank, + key, + ) + return written + + def iter_bank_chunks(opts, bank, count=DEFAULT_KEY_CHUNK_COUNT, key_filter=None): """ Yield ``items`` lists for an arbitrary :class:`salt.cache.Cache` diff --git a/salt/config/__init__.py b/salt/config/__init__.py index b439ba13ce93..4b553686bd75 100644 --- a/salt/config/__init__.py +++ b/salt/config/__init__.py @@ -186,7 +186,7 @@ def _gather_buffer_space(): # total cap across the pool is # ``master_mworker_max_inflight * worker_threads``. Coroutines # blocked on the semaphore create natural TCP / ZMQ backpressure - # — no error return, no dropped requests. + # -- no error return, no dropped requests. "master_mworker_max_inflight": int, # The key fingerprint of the higher-level master for the syndic to verify it is talking to the # intended master @@ -230,9 +230,61 @@ def _gather_buffer_space(): # and can be sync'd as opaque blobs) and joining masters request a # bulk state-sync from an existing peer before becoming Raft voters. "cluster_isolated_filesystem": bool, + # Master switch for cross-master cache replication at the + # ``salt.cache.Cache`` facade. When True (the default when + # ``cluster_id`` is set and ``cluster_isolated_filesystem`` is True) + # every ``cache.store()`` and ``cache.store_many()`` call fans + # out the write to every peer master in the cluster so each + # peer's local cachedir converges on the same state. Peers + # with a shared filesystem (``cluster_isolated_filesystem: + # False``) don't need the fan-out -- they already see each + # other's writes on disk / in mmap. + "cluster_cache_replication": (bool, type(None)), + # Per-bank routing mode for cache writes on a clustered master. + # The value is a dict ``{bank_name: "ring" | "replicate" | + # "none"}``. Banks (or bank prefixes -- longest-prefix match + # on ``/``) that aren't listed default to ``"replicate"`` + # (N-of-N fan-out to every peer). ``"ring"`` delegates the + # write to the HashRing owner (today's + # ``cluster/peer/delegate-write`` path -- 1-of-N). ``"none"`` + # skips replication entirely and is used for banks that own + # their own replication transport (Raft log, state-sync + # chunks). The default-merged entries ``{"jobs": "ring", + # "cluster/consensus": "none", "cluster/state_sync": "none"}`` + # preserve today's job-return behaviour and keep Raft/state-sync + # out of the cache-facade fan-out. + "cluster_cache_routing": dict, + # Maximum number of ``(key, data)`` pairs packed into a single + # ``cluster/runner/delegate_cache_write`` event for + # ``cache.store_many`` batched replication. Larger batches + # amortise the per-event cost but bloat the msgpack payload; + # 1024 fits comfortably under typical bus payload budgets for + # grain-shaped records and keeps catch-up latency bounded when + # a peer is behind. + "cluster_cache_replicate_batch_size": int, + # Join-time catch-up for replicate-mode cache banks. When True + # (the default when ``cluster_cache_replication`` is True) a + # newly-joining master receives every replicate-mode bank's + # current contents as chunked state-sync-chunk events before + # it applies live delegate-cache-write deltas, so writes that + # landed before the join aren't stale on the new peer. + # Operators can set this False to accept eventual-consistency + # catch-up through natural write churn. ``None`` is the + # "unset" sentinel resolved in ``apply_master_config``. + "cluster_state_sync_cache_banks": (bool, type(None)), + # TTL (seconds) for abandoned pending-register windows on the + # master. Chunked-register payloads from a minion that crashes + # mid-stream (sends chunks ``0..k`` then never sends + # ``chunk_final=True``) accumulate per + # ``(minion_id, chunk_scope)`` -- bounded by + # ``N_minions * N_types`` live entries but unbounded if the + # minion crash-loops. The master sweeps on every chunk arrival + # and drops windows with no activity for longer than this TTL, + # logging a WARNING naming the minion. + "cluster_resource_register_pending_ttl": (int, float), # Maximum number of in-memory Raft log entries before the log # compacts into a snapshot. ``None`` (the default) disables - # compaction — fine for small clusters but unbounded growth at + # compaction -- fine for small clusters but unbounded growth at # scale. Setting to a positive integer triggers # ``Log.snapshot()`` whenever the log reaches the threshold; # the snapshot envelope (``raft.snapshot.v1``) carries every @@ -245,7 +297,7 @@ def _gather_buffer_space(): # late joiners that arrive after the cap is hit stay as # non-voting learners indefinitely. Learners still receive # log replication and cluster events, so they remain useful - # for handling minion traffic — they just don't count toward + # for handling minion traffic -- they just don't count toward # election or commit quorum. "cluster_max_voters": (type(None), int), # Auto-replacement of failed voters (Ongaro thesis §6.4 single-server @@ -264,6 +316,18 @@ def _gather_buffer_space(): "cluster_min_voters": int, "cluster_demote_cooldown": float, "cluster_auto_replace_voters": bool, + # Periodic retry controls for ``cluster/peer/discover``. The initial + # fire-and-forget discover event in ``salt.master.Master.start`` can + # be lost silently if the peer's pool-puller is not yet bound on + # port ``cluster_pool_port`` when the event is forwarded. A non- + # founder joining the cluster installs a ``PeriodicCallback`` in + # ``MasterPubServerChannel._schedule_discover_retry`` that re-sends + # discover every ``cluster_join_discover_retry_interval`` seconds + # for peers whose AES key has not yet arrived; the retry cancels + # itself as soon as every peer has replied or after + # ``cluster_join_discover_retry_timeout`` seconds have elapsed. + "cluster_join_discover_retry_interval": (int, float), + "cluster_join_discover_retry_timeout": (int, float), # Use a module function to determine the unique identifier. If this is # set and 'id' is not set, it will allow invocation of a module function # to determine the value of 'id'. For simple invocations without function @@ -541,8 +605,6 @@ def _gather_buffer_space(): "return_retry_tries": int, # Configures amount of retries for Syndic to Master of Masters "syndic_retries": int, - # Top-level pillar key for per-type resource configuration (default: resources) - "resource_pillar_key": str, # Specify one or more returners in which all events will be sent to. Requires that the returners # in question have an event_return(event) function! "event_return": (list, str), @@ -820,6 +882,23 @@ def _gather_buffer_space(): # cachedir under the name of the minion and used to predetermine what minions are expected to # reply from executions. "minion_data_cache": bool, + # Opt-in switch for the persistent mmap-backed grain trie + # index (Mission 4). When ``True`` the master builds a + # scalar-leaf-level index of every grain dict in + # ``resource_grains`` and uses it to answer arbitrary + # ``:`` grain-target predicates without walking + # the bank. See :mod:`salt.utils.mmap_trie_index`. + "enable_grain_trie_index": bool, + # Per-``(path, value)`` sr_key list cap for the trie index. + # Buckets that exceed this size are replaced with a sentinel + # that forces that specific predicate to fall back to the + # slow path, bounding on-disk size and query cost. + "grain_trie_max_list_size": int, + # Streaming-build accumulator threshold for the trie. The + # builder flushes when the aggregate number of pending + # sr_keys across all buckets reaches this value, keeping + # peak builder RAM bounded. + "grain_trie_build_batch_size": int, # The number of seconds between AES key rotations on the master "publish_session": int, # Defines a salt reactor. See https://docs.saltproject.io/en/latest/topics/reactor/ @@ -1420,7 +1499,6 @@ def _gather_buffer_space(): "return_retry_timer": 5, "return_retry_timer_max": 10, "return_retry_tries": 3, - "resource_pillar_key": "resources", "syndic_retries": 3, "random_reauth_delay": 10, "winrepo_source_dir": "salt://win/repo-ng/", @@ -1791,6 +1869,9 @@ def _gather_buffer_space(): "master_job_cache": "local_cache", "job_cache_store_endtime": False, "minion_data_cache": True, + "enable_grain_trie_index": False, + "grain_trie_max_list_size": 100000, + "grain_trie_build_batch_size": 1024, "enforce_mine_cache": False, "ipc_mode": _DFLT_IPC_MODE, "ipc_write_buffer": _DFLT_IPC_WBUFFER, @@ -1976,6 +2057,31 @@ def _gather_buffer_space(): "cluster_pub_fingerprint": None, "cluster_secret": None, "cluster_isolated_filesystem": False, + # ``cluster_cache_replication`` default is resolved in + # ``apply_master_config`` because it depends on whether + # ``cluster_id`` is configured and whether the deployment is + # using an isolated per-master filesystem. Keep the raw + # default here as False (non-clustered masters). + # ``None`` is the "unset" sentinel. ``apply_master_config`` + # resolves it to True when ``cluster_id`` AND + # ``cluster_isolated_filesystem`` are set, else False. + # Operators can force True/False explicitly in the master + # config. + "cluster_cache_replication": None, + # Per-bank routing dict; ``"jobs": "ring"`` default is merged + # in ``apply_master_config`` so operator overrides can still + # expand or override it. See ``cluster_cache_replication``. + "cluster_cache_routing": {}, + "cluster_cache_replicate_batch_size": 1024, + # Join-time cache-bank catch-up default -- resolved in + # ``apply_master_config`` to match ``cluster_cache_replication``. + "cluster_state_sync_cache_banks": None, + # Default pending-register-window TTL mirrors the module + # constant :data:`salt.utils.resource_registry.DEFAULT_PENDING_REGISTERS_TTL` + # (300 s / 5 min). Operators sizing clusters with larger + # 200K+ registers can increase this if normal per-minion + # windows approach the default. + "cluster_resource_register_pending_ttl": 300.0, "cluster_max_log_size": None, "cluster_max_voters": None, "cluster_voter_health_check_interval": 1.0, @@ -1983,6 +2089,8 @@ def _gather_buffer_space(): "cluster_min_voters": 3, "cluster_demote_cooldown": 60.0, "cluster_auto_replace_voters": False, + "cluster_join_discover_retry_interval": 2, + "cluster_join_discover_retry_timeout": 60, "features": {}, "publish_signing_algorithm": "PKCS1v15-SHA1", "cluster_encryption_algorithm": "OAEP-SHA1", @@ -4452,6 +4560,57 @@ def master_config( return salt.utils.optsdict.OptsDict.from_dict(opts, name="master_config") +#: Built-in per-bank cache routing defaults. Operator-supplied entries +#: in ``cluster_cache_routing`` win over these; banks absent from both +#: default to ``"replicate"`` at the call site. ``"jobs"`` is pinned to +#: ``"ring"`` so today's single-owner job-return behaviour is preserved +#: on upgrade (backward compat). +#: +#: The lookup uses longest-prefix match on the ``/``-delimited bank name +#: (``salt/cache/__init__.py:_resolve_bank_routing``), so these keys +#: match any compound bank below them -- e.g. ``jobs`` catches +#: ``jobs/loads``, ``jobs/returns/``, and ``cluster/consensus`` +#: catches both ``cluster/consensus//cluster`` and +#: ``cluster/consensus//cluster/log`` which Raft writes to. +#: +#: ``"none"`` means "skip replication entirely" -- used for banks whose +#: contents are already replicated by a dedicated transport (Raft log +#: for ``cluster/consensus``, state-sync chunking for +#: ``cluster/state_sync``) and MUST NOT be fanned out via the cache +#: facade, since the peer's handler would overwrite its own authoritative +#: copy with another master's state. +_CLUSTER_CACHE_ROUTING_DEFAULTS = { + "jobs": "ring", + "cluster/consensus": "none", + "cluster/state_sync": "none", +} + + +def get_cluster_cache_routing(opts): + """ + Return the effective per-bank cache routing for this master. + + Merges operator-supplied ``opts["cluster_cache_routing"]`` with the + built-in defaults. Operator entries win; built-in entries from + ``_CLUSTER_CACHE_ROUTING_DEFAULTS`` (``jobs``, ``cluster/consensus``, + ``cluster/state_sync``) are injected only when the operator has not + supplied an entry for the same key. ``"none"`` entries must survive + the merge since they are the opt-out marker for banks that own their + own replication transport. + + Banks absent from the result default to ``"replicate"`` at the read + site (``salt/cache/__init__.py:_resolve_bank_routing`` -- longest- + prefix match). + + :param opts: Master opts dict. + :returns: Dict of ``{bank_prefix: "ring"|"replicate"|"none"}``. + """ + merged = dict(opts.get("cluster_cache_routing") or {}) + for _bank, _mode in _CLUSTER_CACHE_ROUTING_DEFAULTS.items(): + merged.setdefault(_bank, _mode) + return merged + + def apply_master_config(overrides=None, defaults=None): """ Returns master configurations dict. @@ -4609,6 +4768,33 @@ def apply_master_config(overrides=None, defaults=None): log.warning("Cluster pki dir defined without a cluster_id, ignoring.") opts["cluster_pki_dir"] = None + # Resolve cache-replication defaults. The raw DEFAULT_MASTER_OPTS + # value is False (non-clustered masters never fan out); flip the + # default to True on clustered masters running with an isolated + # per-member filesystem, since that's where peers can't see each + # other's writes through the cachedir. Operators can still force + # False explicitly in the master config. + if ( + "cluster_cache_replication" not in opts + or opts["cluster_cache_replication"] is None + ): + opts["cluster_cache_replication"] = bool( + opts.get("cluster_id") and opts.get("cluster_isolated_filesystem") + ) + # Join-time cache-bank catch-up follows the same master-switch -- + # enabled when replication is on, operator can force False to + # opt out of the join-time bandwidth. + if ( + "cluster_state_sync_cache_banks" not in opts + or opts["cluster_state_sync_cache_banks"] is None + ): + opts["cluster_state_sync_cache_banks"] = bool( + opts.get("cluster_cache_replication") + ) + # Merge the default per-bank routing map via the public resolver so + # the merge rule has a single source of truth. + opts["cluster_cache_routing"] = get_cluster_cache_routing(opts) + # Enabling open mode requires that the value be set to True, and # nothing else! opts["open_mode"] = opts["open_mode"] is True diff --git a/salt/master.py b/salt/master.py index f30f656dff4b..a5bafddb3154 100644 --- a/salt/master.py +++ b/salt/master.py @@ -2894,56 +2894,179 @@ def __register_resources_sync(self, load): # cache (:conf_master:`minion_data_cache`). Registration must always run # when minions report inventory; otherwise bare-id / T@ targeting breaks # silently while still returning success to the minion. - n_put, n_del = salt.utils.minions.update_resource_index( - self.opts, load["id"], load["resources"] - ) - log.debug( - "Registered resources for minion '%s': %s (put=%d, deleted=%d)", - load["id"], - list(load["resources"].keys()), - n_put, - n_del, + # + # Two cross-master register paths depending on ring configuration: + # + # * Standalone or broadcast (no ``resource_index`` ring configured): + # write the full set locally here, fire a single broadcast + # replication event; every peer applies the full set on arrival. + # + # * Ring-routed (operator ran ``salt-run cluster.ring_create + # resource_index`` + ``cluster.route_set resource_index ring``): + # the ring state only lives in the publish-daemon process (its + # per-ring Raft groups aren't visible across the MWorker fork + # boundary -- see ``salt/cluster/ring_membership.py`` module + # docstring). To avoid a double-write of peer-owned SRNs, + # skip the local MWorker write entirely when the cluster is + # healthy and defer both the local-owner shard write and the + # peer fan-out to ``_run_delegate_resource_register_write`` + # in the publish daemon, which has ring access. + opts = self.opts or {} + cluster_mode = bool( + opts.get("cluster_id") + and opts.get("cluster_isolated_filesystem") + and opts.get("cluster_cache_replication") is not False ) + # Chunked-register metadata: the minion-side register loop ships + # resource inventory in per-type chunks tagged with + # ``chunk_rtype`` / ``chunk_index`` / ``chunk_final``. Only the + # terminal chunk per type triggers the stale-SRN prune; mid-stream + # chunks ``put_many`` without a delete-diff so an earlier chunk + # of the same window is never prematurely pruned. Legacy single- + # shot callers that omit these fields fall through to ``__all__`` + # scope with ``chunk_final=True`` -- identical semantics to the + # pre-chunk code path. + chunk_rtype = load.get("chunk_rtype") + chunk_final = load.get("chunk_final", True) + chunk_index = load.get("chunk_index") + chunk_scope = chunk_rtype if chunk_rtype else "__all__" + if not cluster_mode: + n_put, n_del = salt.utils.minions.update_resource_index_chunk( + self.opts, + load["id"], + load["resources"], + chunk_final, + chunk_scope=chunk_scope, + chunk_index=chunk_index, + ) + log.debug( + "Registered resources for minion '%s' scope=%s final=%s: " + "%s (put=%d, deleted=%d)", + load["id"], + chunk_scope, + chunk_final, + list(load["resources"].keys()), + n_put, + n_del, + ) + else: + log.debug( + "resource-registry: deferring local + peer register for " + "minion '%s' scope=%s final=%s to publish-daemon via " + "replication event", + load["id"], + chunk_scope, + chunk_final, + ) + # Cross-master replication: fire a cluster-runner event the publish + # daemon intercepts. ``_run_delegate_resource_register_write`` + # consults ``ring_membership.get_ring("resource_index")`` and + # either partitions by owner (ring-routed) or fans to every peer + # (broadcast fallback). In cluster mode the daemon also owns + # the local-shard write so the MWorker here doesn't double-write. + # ``chunk_final`` + ``chunk_scope`` + ``chunk_index`` are + # propagated through the event so peers (and the local + # publish-daemon writer) honor the same chunk-aggregation + # semantics and the out-of-order gap check. + if cluster_mode: + try: + from salt.cache._cluster_replicator import ( # pylint: disable=import-outside-toplevel + fire_resource_register_write, + ) + + fire_resource_register_write( + opts, + load["id"], + load["resources"], + chunk_final=chunk_final, + chunk_scope=chunk_scope, + chunk_index=chunk_index, + ) + except Exception: # pylint: disable=broad-except + log.exception( + "resource-replicate: failed to fan out register for %s", + load["id"], + ) fire_event = False # Persist per-resource grains in the ``resource_grains`` cache bank # so ``salt -G ':' …`` can match resources alongside - # minions. Stale entries (resource removed from this minion since - # last registration) are flushed first so a shrinking inventory - # doesn't leave ghost entries. + # minions. The ghost-entry flush (SRNs that disappeared from a + # shrinking inventory) only runs on the terminal chunk of the + # window -- mid-stream flushes would wipe grains stored by the + # chunk-N-1 handler for SRNs present only in chunk N. if self.opts.get("minion_data_cache", False): resource_grains = load.get("resource_grains") or {} try: cache = self.masterapi.cache + # Always store this chunk's grains (additive). Use + # ``store_many`` so the whole chunk fires a single batched + # ``delegate-cache-write`` event (bounded by + # ``cluster_cache_replicate_batch_size``) instead of one + # event per SR. current_srns = set(resource_grains.keys()) - # Walk existing entries and drop ones tied to this minion - # that aren't in the new payload. Owner identification piggy - # backs on the registry: each SRN is owned by exactly one - # minion at a time. - for srn in list( - cache.list(salt.utils.resource_registry.RESOURCE_GRAINS_BANK) or [] - ): - rtype, _, rid = srn.partition(":") - if not rid: - continue - if srn in current_srns: + batch = [] + for srn, gdict in resource_grains.items(): + if not isinstance(gdict, dict): continue - owners = self.ckminions.registry.get_managing_minions_for_srn( - rtype, rid + # Enforce the ``resource_id`` convention structurally: + # the id portion of the bank key is authoritative, so + # any grain whose module omitted or diverged on + # ``resource_id`` is normalised here. Grain targeting's + # ``resource_id:`` fast path relies on this + # invariant to skip the per-match fetch. + _rtype, _sep, _rid = srn.partition(":") + if _sep and _rid: + gdict["resource_id"] = _rid + batch.append((srn, gdict)) + if batch: + cache.store_many( + salt.utils.resource_registry.RESOURCE_GRAINS_BANK, + batch, ) - if load["id"] in owners or not owners: + if chunk_final: + # Terminal chunk: by this point + # ``register_minion_chunk`` has already committed the + # full window's SRNs to the registry, so the registry + # authoritatively knows which SRNs ``load["id"]`` + # owns right now. Flush any ``resource_grains`` SRN + # of a type this window scoped that the registry + # shows as orphaned (no minion owns it). SRNs still + # owned by anyone -- this minion included -- are + # kept. + for srn in list( + cache.list(salt.utils.resource_registry.RESOURCE_GRAINS_BANK) + or [] + ): + rtype, _, rid = srn.partition(":") + if not rid: + continue + if chunk_scope != "__all__" and rtype != chunk_scope: + # Scoped terminal chunk: only prune SRNs of + # the type this window is replacing. + continue + if srn in current_srns: + # Just stored by this terminal chunk -- keep. + continue + owners = self.ckminions.registry.get_managing_minions_for_srn( + rtype, rid + ) + if owners: + # Still owned by someone (incl. this minion + # -- an earlier chunk of the current window). + continue + # No owner in the registry -- orphan, safe to + # flush. try: cache.flush( - salt.utils.resource_registry.RESOURCE_GRAINS_BANK, srn + salt.utils.resource_registry.RESOURCE_GRAINS_BANK, + srn, ) except Exception as exc: # pylint: disable=broad-except - log.debug("resource_grains flush %s failed: %s", srn, exc) - for srn, gdict in resource_grains.items(): - if isinstance(gdict, dict): - cache.store( - salt.utils.resource_registry.RESOURCE_GRAINS_BANK, - srn, - gdict, - ) + log.debug( + "resource_grains flush %s failed: %s", + srn, + exc, + ) except Exception as exc: # pylint: disable=broad-except log.warning( "Failed to persist resource_grains for minion '%s': %s", @@ -2952,8 +3075,10 @@ def __register_resources_sync(self, load): ) # Signal the caller to fire the cache-refresh event on the # async event bus. ``_pillar`` fires the analogous event when - # ordinary minion grains are refreshed in the cache. - if self.opts.get("minion_data_cache_events") is True: + # ordinary minion grains are refreshed in the cache. Only + # fire on terminal chunks to avoid burst-firing N refresh + # events for an N-chunk register. + if chunk_final and self.opts.get("minion_data_cache_events") is True: fire_event = True return True, fire_event, load["id"] if fire_event else None @@ -3655,57 +3780,129 @@ def _sync_mine_flush(self, load): return self.masterapi._mine_flush(load, skip_verify=True) def _sync_register_resources(self, load): - """Sync-shim body of ``_register_resources`` (pre-PR verbatim).""" + """Sync-shim body of ``_register_resources``. + + Chunk-aggregation parity: carries the same ``chunk_rtype`` / + ``chunk_final`` plumbing as :meth:`__register_resources_sync` so + the MWorker-sync entry point doesn't under-aggregate multi-chunk + registers (which would prune earlier chunks mid-stream). + """ load = self._AESFuncs__verify_load(load, ("id", "resources")) if load is False: return {} - n_put, n_del = salt.utils.minions.update_resource_index( - self.opts, load["id"], load["resources"] - ) - log.debug( - "Registered resources for minion '%s': %s (put=%d, deleted=%d)", - load["id"], - list(load["resources"].keys()), - n_put, - n_del, + opts = self.opts or {} + cluster_mode = bool( + opts.get("cluster_id") + and opts.get("cluster_isolated_filesystem") + and opts.get("cluster_cache_replication") is not False ) + chunk_rtype = load.get("chunk_rtype") + chunk_final = load.get("chunk_final", True) + chunk_index = load.get("chunk_index") + chunk_scope = chunk_rtype if chunk_rtype else "__all__" + if not cluster_mode: + n_put, n_del = salt.utils.minions.update_resource_index_chunk( + self.opts, + load["id"], + load["resources"], + chunk_final, + chunk_scope=chunk_scope, + chunk_index=chunk_index, + ) + log.debug( + "Registered resources for minion '%s' scope=%s final=%s: " + "%s (put=%d, deleted=%d)", + load["id"], + chunk_scope, + chunk_final, + list(load["resources"].keys()), + n_put, + n_del, + ) + else: + log.debug( + "resource-registry: deferring local + peer register for " + "minion '%s' scope=%s final=%s to publish-daemon via " + "replication event", + load["id"], + chunk_scope, + chunk_final, + ) + try: + from salt.cache._cluster_replicator import ( # pylint: disable=import-outside-toplevel + fire_resource_register_write, + ) + + fire_resource_register_write( + opts, + load["id"], + load["resources"], + chunk_final=chunk_final, + chunk_scope=chunk_scope, + chunk_index=chunk_index, + ) + except Exception: # pylint: disable=broad-except + log.exception( + "resource-replicate: failed to fan out register for %s", + load["id"], + ) if self.opts.get("minion_data_cache", False): resource_grains = load.get("resource_grains") or {} try: cache = self.masterapi.cache current_srns = set(resource_grains.keys()) - for srn in list( - cache.list(salt.utils.resource_registry.RESOURCE_GRAINS_BANK) or [] - ): - rtype, _, rid = srn.partition(":") - if not rid: - continue - if srn in current_srns: + batch = [] + for srn, gdict in resource_grains.items(): + if not isinstance(gdict, dict): continue - owners = self.ckminions.registry.get_managing_minions_for_srn( - rtype, rid + # Enforce the ``resource_id`` convention structurally: + # the id portion of the bank key is authoritative, so + # any grain whose module omitted or diverged on + # ``resource_id`` is normalised here. Grain targeting's + # ``resource_id:`` fast path relies on this + # invariant to skip the per-match fetch. + _rtype, _sep, _rid = srn.partition(":") + if _sep and _rid: + gdict["resource_id"] = _rid + batch.append((srn, gdict)) + if batch: + cache.store_many( + salt.utils.resource_registry.RESOURCE_GRAINS_BANK, + batch, ) - if load["id"] in owners or not owners: + if chunk_final: + # Terminal chunk: see __register_resources_sync for + # the matching narrative -- orphans only, scoped. + for srn in list( + cache.list(salt.utils.resource_registry.RESOURCE_GRAINS_BANK) + or [] + ): + rtype, _, rid = srn.partition(":") + if not rid: + continue + if chunk_scope != "__all__" and rtype != chunk_scope: + continue + if srn in current_srns: + continue + owners = self.ckminions.registry.get_managing_minions_for_srn( + rtype, rid + ) + if owners: + continue try: cache.flush( - salt.utils.resource_registry.RESOURCE_GRAINS_BANK, srn + salt.utils.resource_registry.RESOURCE_GRAINS_BANK, + srn, ) except Exception as exc: # pylint: disable=broad-except log.debug("resource_grains flush %s failed: %s", srn, exc) - for srn, gdict in resource_grains.items(): - if isinstance(gdict, dict): - cache.store( - salt.utils.resource_registry.RESOURCE_GRAINS_BANK, - srn, - gdict, - ) except Exception as exc: # pylint: disable=broad-except log.warning( "Failed to persist resource_grains for minion '%s': %s", load["id"], exc, ) - if self.opts.get("minion_data_cache_events") is True: + if chunk_final and self.opts.get("minion_data_cache_events") is True: self.event.fire_event( {"Resource cache refresh": load["id"]}, tagify(load["id"], "refresh", "resource"), diff --git a/salt/matchers/resource_match.py b/salt/matchers/resource_match.py index cb6ddc7bb378..70e9461b04af 100644 --- a/salt/matchers/resource_match.py +++ b/salt/matchers/resource_match.py @@ -9,10 +9,19 @@ T@vcf_host # any resource of this type T@vcf_host:esxi-01 # one specific resource by SRN -This matcher is evaluated on the minion. It reads from ``opts["resources"]``, -which is populated when the minion loads its resource modules — analogous to -how ``grain_match`` reads from ``opts["grains"]``. No cache or registry -lookup is performed. +This matcher is evaluated on the minion. It consults the configured +``resource_types`` opt (via :func:`salt.utils.resources.enabled_resource_types`) +and, when a specific SRN is being matched, iterates that type's +``discover(opts)`` generator to check membership. No inventory dict is +materialized on the minion -- the memory-ceiling invariant from +``feedback_minion_never_holds_full_resource_set.md`` is preserved. + +The pre-refactor shape read from ``opts["resources"]``, which was +populated at minion startup by the module loader. That accumulator +was removed when pillar/grain-driven resource inventory was dropped +(a 200K-resource dict on the minion blew the memory ceiling); the +matcher was never ported to the new generator-based API. This module +is that port. """ import logging @@ -20,6 +29,47 @@ log = logging.getLogger(__name__) +def _iter_resource_ids(opts, resource_type): + """ + Yield bare resource IDs for ``resource_type`` by calling the + module's ``discover(opts)`` generator. + + Returns an empty iterator when the loader is unavailable or the + type has no ``discover`` entry -- matcher callers treat that as + "no managed resources of this type", which is the correct + answer for an un-configured type. + """ + try: + import salt.loader # noqa: PLC0415 + except Exception as exc: # pylint: disable=broad-except + log.debug("resource_match: salt.loader unavailable (%s)", exc) + return + try: + funcs = salt.loader.resource(opts) + except Exception as exc: # pylint: disable=broad-except + log.debug( + "resource_match: resource loader failed for '%s' (%s)", + resource_type, + exc, + ) + return + discover_fn = f"{resource_type}.discover" + if discover_fn not in funcs: + log.debug( + "resource_match: no discover function for type '%s'", + resource_type, + ) + return + try: + ids = funcs[discover_fn](opts) + except Exception as exc: # pylint: disable=broad-except + log.debug("resource_match: discover(%s) failed: %s", resource_type, exc) + return + if ids is None: + return + yield from ids + + def match(tgt, opts=None, minion_id=None): """ Return ``True`` if this minion manages at least one resource that matches @@ -27,30 +77,43 @@ def match(tgt, opts=None, minion_id=None): ``tgt`` is the portion of the ``T@`` expression after the ``@``. It is either a bare resource type (``vcf_host``) or a full SRN - (``vcf_host:esxi-01``). When a bare type is given, every resource of that - type in ``opts["resources"]`` satisfies the match. When a full SRN is - given, only an exact match against a resource ID in ``opts["resources"]`` - satisfies it. - - The structure of ``opts["resources"]`` is populated by the resource module - loader at minion startup, analogous to ``opts["grains"]``. + (``vcf_host:esxi-01``). When a bare type is given, the match succeeds iff + the type is in ``resource_types`` and its ``discover`` generator yields at + least one ID. When a full SRN is given, the match iterates the type's + ``discover`` generator and short-circuits on the first match (so + matching does not materialize the full inventory). - :param str tgt: The T@ pattern — a resource type or a full SRN. + :param str tgt: The T@ pattern -- a resource type or a full SRN. :param dict opts: Salt opts dict; defaults to ``__opts__``. :param str minion_id: The minion ID to evaluate; defaults to ``opts["id"]``. :rtype: bool """ if opts is None: opts = __opts__ # pylint: disable=undefined-variable - resources = opts.get("resources", {}) - if not resources: - return False + + import salt.utils.resources # noqa: PLC0415 + + enabled = set(salt.utils.resources.enabled_resource_types(opts)) if ":" in tgt: resource_type, resource_id = tgt.split(":", 1) - result = resource_id in resources.get(resource_type, []) + if resource_type not in enabled: + result = False + else: + result = False + for rid in _iter_resource_ids(opts, resource_type): + if rid == resource_id: + result = True + break else: - result = bool(resources.get(tgt)) + if tgt not in enabled: + result = False + else: + # Bare-type match: True iff the generator yields at least one ID. + result = False + for _rid in _iter_resource_ids(opts, tgt): + result = True + break - log.debug("resource_match: T@%s => %s (resources=%s)", tgt, result, list(resources)) + log.debug("resource_match: T@%s => %s (enabled=%s)", tgt, result, sorted(enabled)) return result diff --git a/salt/minion.py b/salt/minion.py index 4ffd8f98ecb8..23560b206fae 100644 --- a/salt/minion.py +++ b/salt/minion.py @@ -958,10 +958,13 @@ def gen_modules(self, initial_load=False, context=None): finally: pillar.destroy() - # Populate opts["resources"] from pillar now that pillar is available. - # Must happen before the resource loader loop below so that per-type - # execution module loaders are created for the correct set of types. - self.opts["resources"] = self._discover_resources() + # Resource inventory is NOT materialized into ``opts["resources"]`` + # anymore -- the full ID list is streamed from ``discover()`` + # inside :meth:`_register_resources_with_master` and consumed in + # chunks. The configured type list (from the ``resource_types`` + # minion opt) drives per-type loader construction and ``init()`` + # dispatch below. + enabled_types = salt.utils.resources.enabled_resource_types(self.opts) self.utils = salt.loader.utils(self.opts, context=context) functions = salt.loader.minion_mods( @@ -996,7 +999,7 @@ def gen_modules(self, initial_load=False, context=None): # assignment is atomic in CPython, so the old loaders remain visible # until the new complete set is ready. _new_resource_loaders = {} - for resource_type in self.opts.get("resources", {}): + for resource_type in enabled_types: rtype_base = ( f"{self.opts.get('loaded_base_name', 'salt.loaded.int')}" f".resource.{resource_type}" @@ -1014,7 +1017,7 @@ def gen_modules(self, initial_load=False, context=None): # Call init() on each resource type so that __context__ is populated # before any per-resource operations (grains, ping, etc.) are dispatched. # Mirrors how proxy.init() is called during proxy-minion startup. - for resource_type in self.opts.get("resources", {}): + for resource_type in enabled_types: init_fn = f"{resource_type}.init" if init_fn in self.resource_funcs: try: @@ -1068,62 +1071,52 @@ def _reload_modules(self, initial_load=False, context=None): # modules and return None instead. self.gen_modules(initial_load=initial_load, context=context) - def _discover_resources(self): + def _iter_resource_ids(self, resource_type): """ - Build ``opts["resources"]`` by calling each resource type's - ``discover(opts)`` function. + Yield bare resource IDs for ``resource_type`` by invoking that + type's ``discover(opts)`` function. - Resource types are read from the pillar subtree at - ``opts["pillar"][opts["resource_pillar_key"]]`` (default key - ``"resources"``, configurable via minion option ``resource_pillar_key``). - A temporary resource loader is used to call each type's - ``discover(opts)``; the return value is a dict of - ``{resource_type: [resource_id, ...]}``. + The returned iterable is consumed once per call: the register + loop uses this to stream fixed-size chunks (see + :meth:`_register_resources_with_master`); job-dispatch + resolvers call it per-dispatch and iterate briefly. The full + ID list is never kept resident -- Salt's minion-memory-ceiling + invariant is enforced structurally by never calling ``list()`` + on this iterator from framework code. - If the merged pillar contains no key by that name, that is treated the - same as an empty mapping: no pillar-declared resource types, so - discovery returns an empty dict (no stale IDs left in - ``opts["resources"]``). - - If the pillar *does* contain that key (even if its value is empty / - all entries removed), that is an authoritative declaration and the - result reflects only what the pillar says (via ``discover()`` per - type). - - Called from :meth:`gen_modules` after pillar is compiled and before - the per-type execution-module loaders are created. + A type whose loader has no ``.discover`` entry, or whose + ``discover()`` raises, yields nothing (with a logged warning). """ - pillar_resources = salt.utils.resources.pillar_resources_tree(self.opts) - - # A minimal resource loader is sufficient here — discover() only reads - # from the opts dict passed to it and does not need other dunders. - discovery_loader = salt.loader.resource(self.opts) - discovered = {} - for resource_type in pillar_resources: - discover_fn = f"{resource_type}.discover" - if discover_fn not in discovery_loader: - log.warning( - "No resource module found for type '%s'; skipping discovery.", - resource_type, - ) - continue + funcs = getattr(self, "resource_funcs", None) + if not funcs: try: - ids = discovery_loader[discover_fn](self.opts) - if ids: - discovered[resource_type] = list(ids) - log.debug( - "Discovered %d resource(s) of type '%s': %s", - len(ids), - resource_type, - ids, - ) + funcs = salt.loader.resource(self.opts) except Exception as exc: # pylint: disable=broad-except log.warning( - "Resource discovery failed for type '%s': %s", + "Resource loader unavailable for type '%s': %s", resource_type, exc, ) - return discovered + return + discover_fn = f"{resource_type}.discover" + if discover_fn not in funcs: + log.warning( + "No resource module found for type '%s'; skipping discovery.", + resource_type, + ) + return + try: + ids = funcs[discover_fn](self.opts) + except Exception as exc: # pylint: disable=broad-except + log.warning( + "Resource discovery failed for type '%s': %s", + resource_type, + exc, + ) + return + if ids is None: + return + yield from ids @staticmethod def process_schedule(minion, loop_interval): @@ -2528,11 +2521,9 @@ async def _post_master_init(self, master): # the pyzmq Context.__del__ wedge that sync ``destroy`` # can leave behind (see PR-70316 trace). await async_pillar.aclose() - # _setup_core uses _load_modules only — unlike gen_modules it does not - # run _discover_resources(). tune_in schedules _register_resources_with_master - # right after connect; without this, the master registry gets {} until an - # async pillar refresh completes (easy to miss with saltutil.sync_all). - self.opts["resources"] = self._discover_resources() + # Resource types are configured via ``resource_types`` opt and + # materialized lazily by the register loop; no eager discovery + # happens here (see :meth:`_register_resources_with_master`). if not self.ready: self._setup_core() @@ -2545,7 +2536,9 @@ async def _post_master_init(self, master): self.function_errors, self.executors, ) = self._load_modules() - self.opts["resources"] = self._discover_resources() + # Resource types are opt-driven and unchanged by a pillar + # refresh; the chunked register path re-streams IDs when + # ``resource_refresh`` fires. if hasattr(self, "schedule"): self.schedule.functions = self.functions self.schedule.returners = self.returners @@ -4624,26 +4617,36 @@ async def _fire_master_minion_start(self): include_startup_grains=include_grains, ) - def _collect_resource_grains(self): + def _collect_resource_grains(self, resources=None): """ - Render per-resource grain dicts for every resource this minion manages. + Render per-resource grain dicts for the given resource set. - For each ``(resource_type, resource_id)`` pair, sets the per-call - :data:`salt.loader.context.resource_ctxvar` and invokes - ``resource_funcs[f"{type}.grains"]()``. Returns a mapping keyed by - composite SRN ``":"`` -> grain dict, suitable for shipping - to the master in :meth:`_register_resources_with_master`. + ``resources`` is a mapping ``{resource_type: [resource_id, ...]}`` + -- normally one type's chunk at a time, supplied by + :meth:`_register_resources_with_master`. When called without + arguments the method streams every enabled type by iterating + each type's ``discover()`` in turn; it is up to the caller to + bound the working set (target resolution calls per-type). - Resource types without a ``grains`` callable are skipped silently. - Per-resource grain failures are logged and skipped; a single broken - resource never blocks registration of the others. + Returns a mapping keyed by composite SRN ``":"`` -> + grain dict, suitable for shipping to the master. Resource types + without a ``grains`` callable are skipped silently; per-resource + grain failures are logged and skipped so a single broken + resource never blocks the siblings. :rtype: dict[str, dict] """ import salt.loader.context as _loader_ctx # noqa: PLC0415 + if resources is None: + # Streaming form: iterate each enabled type's discover() so the + # caller never has to pre-build an inventory dict. Used by the + # target-resolution grain cache fallback. + resources = {} + for rtype in salt.utils.resources.enabled_resource_types(self.opts): + resources[rtype] = list(self._iter_resource_ids(rtype)) + resource_grains = {} - resources = self.opts.get("resources", {}) for rtype, rids in (resources or {}).items(): grains_fn = f"{rtype}.grains" if grains_fn not in getattr(self, "resource_funcs", {}): @@ -4669,81 +4672,149 @@ def _collect_resource_grains(self): async def _register_resources_with_master(self): """ - Send this minion's resource list to the master for registry population. - - Called on startup (and reconnect) so that the master's - ``minion_resources`` cache bank is up-to-date. This allows - :class:`salt.utils.minions.CkMinions` to include resource IDs when - expanding glob / non-compound targets (e.g. ``salt '*' test.ping``). - - Also ships a per-resource ``resource_grains`` mapping so the master - can populate its ``resource_grains`` cache bank for ``-G`` / - ``salt -G`` grain-based targeting of resources. - - **Freshness model.** The ``resource_grains`` snapshot is taken at - the moment of this call by :meth:`_collect_resource_grains`. A - per-resource ``.grains_refresh()`` invocation that mutates - the underlying state does **not** automatically propagate to the - master — the master's view is refreshed only when this method runs - again. The triggers that re-run it are: + Register this minion's resources with the master. + + Resource IDs are streamed per-type from each enabled resource + module's ``discover(opts)`` generator -- the minion never holds + the full inventory in an ``opts["resources"]`` mapping. For + each enabled type, IDs are drained from the generator into a + chunk of size :conf_minion:`resource_register_chunk_size` + (default 10000) and shipped to the master; the chunk is then + dropped so the memory footprint for a 200K-resource type stays + bounded by ``chunk_size * per-SRN-cost`` rather than + ``total * per-SRN-cost`` (see memory-ceiling invariant). + + Per-resource grains are rendered per-chunk inside + :meth:`_send_register_chunk` and shipped with that chunk's + SRNs. Grain dicts are NOT accumulated across chunks on the + minion -- the chunk's grains are referenced only by the + outgoing RPC load and are dropped when the chunk completes. + This enforces the memory-ceiling invariant: a 200K-resource + register holds at most ``chunk_size`` grain dicts resident at + any instant instead of all 200K. + + The master's ``resource_grains`` bank sees a fresh snapshot + every time this method runs; the triggers are: * minion start / reconnect (``tune_in``); - * the ``resource_refresh`` event on the minion event bus (see - ``manage_event_iter``); and - * a successful ``saltutil.refresh_pillar`` / ``module_refresh`` - (because :meth:`_post_master_init` re-discovers resources). - - Operators who need the master to see fresh resource grains after - an out-of-band state change should fire ``resource_refresh`` or - run ``salt-call saltutil.refresh_pillar``. - - An empty resource dict is sent deliberately when the minion has no - resources — this clears any stale entries left by a previous - registration (e.g. after a resource type is removed from the pillar). - """ - resources = self.opts.get("resources", {}) - if resources and not getattr(self, "resource_funcs", None): - # ``resource_funcs`` is the loader for ``salt/resource/.py`` - # connection modules. Unlike ``functions``/``returners`` (built - # by :meth:`_setup_core` during :meth:`_post_master_init`), the - # resource loaders are normally materialised lazily on the - # **first job dispatch** by :meth:`_thread_return.gen_modules`. - # Resource registration runs **before** any job has been - # dispatched, so without this eager call ``resource_funcs`` is - # an empty :class:`LazyLoader` and - # :meth:`_collect_resource_grains` finds no ``.grains`` - # callables to invoke — the master ends up with an empty - # ``resource_grains`` bank and ``salt -G ...`` silently fails - # to match resources. The cost is paid once per registration; - # subsequent calls hit the populated loader and skip this - # branch. + * the ``resource_refresh`` event on the minion event bus; + * a successful ``saltutil.refresh_pillar`` / ``module_refresh``. + + A final chunk with ``chunk_final=True`` is sent per type (and a + type=None chunk when the minion manages no types at all) so the + master-side registry can prune stale entries left by a previous + registration. + """ + # ``self.tok`` is assigned on a successful auth handshake + # (see ``connect_master``). Early callers -- e.g. a + # ``pillar_refresh`` fired before the first auth round-trip + # has completed, a proxy minion that has not yet attached its + # pub channel, or a bare ``Minion`` built for a unit test -- + # may not have it yet. Rather than crash with AttributeError + # inside ``_send_register_chunk``, bail: the register will be + # retried on the next pillar_refresh / tune_in once auth has + # populated ``tok``. + if getattr(self, "tok", None) is None: + log.debug("Skipping resource register: auth token not yet available") + return + enabled_types = salt.utils.resources.enabled_resource_types(self.opts) + chunk_size = max(int(self.opts.get("resource_register_chunk_size", 10000)), 1) + + if enabled_types and not getattr(self, "resource_funcs", None): + # Resource loaders are normally built lazily on first job + # dispatch; registration runs earlier, so eagerly rebuild + # modules here or the discover + grains callables are + # missing from ``resource_funcs``. try: self.gen_modules() except Exception as exc: # pylint: disable=broad-except log.warning( - "Failed to gen_modules before resource grain collection: %s", + "Failed to gen_modules before resource registration: %s", exc, ) - resource_grains = self._collect_resource_grains() if resources else {} - # Cache locally so :meth:`_resolve_resource_targets` can resolve - # ``tgt_type == "grain"`` without re-rendering. - self._resource_grains_cache = resource_grains + + sent_any = False + for rtype in enabled_types: + chunk_ids = [] + chunk_index = 0 + for rid in self._iter_resource_ids(rtype): + chunk_ids.append(rid) + if len(chunk_ids) >= chunk_size: + # Ship the full chunk, then drop it so the per- + # chunk grain dict rendered inside + # ``_send_register_chunk`` can be GC-ed before the + # next chunk is accumulated. No cross-chunk + # grain-cache accumulation. + await self._send_register_chunk( + rtype, chunk_ids, chunk_index, final=False + ) + chunk_ids = [] + chunk_index += 1 + # Final chunk per type (possibly empty). Always sent so the + # master can prune stale SRNs. + await self._send_register_chunk(rtype, chunk_ids, chunk_index, final=True) + sent_any = True + # Drop the final chunk reference so the per-type peak is + # released before iterating the next type. + del chunk_ids + + if not sent_any: + # No enabled types -- still send a final empty chunk so the + # master clears any stale state from a previous config. + await self._send_register_chunk(None, [], 0, final=True) + + # Explicitly clear any stale ``_resource_grains_cache`` left + # by a prior (pre-refactor) register. We deliberately do NOT + # populate it here -- holding the full inventory's grain dict + # on the minion violates the memory-ceiling invariant. + # :meth:`_resolve_resource_targets` renders grains on-demand + # per-type when a job is dispatched (bounded by one type's + # chunk). + self._resource_grains_cache = None + + async def _send_register_chunk(self, rtype, ids, index, final): + """ + Ship one chunk of the register payload to the master. + + A chunked RPC per ``(rtype, index)`` carries only the IDs in + that chunk plus the grains for them. Chunk sequencing is + communicated via the ``chunk_rtype``/``chunk_index``/ + ``chunk_final`` fields; a master that ignores these fields + simply applies each chunk as a standalone registration -- for + a multi-chunk type the per-chunk writes are additive against + the mmap registry. ``chunk_final=True`` is the signal for + stale-entry pruning for ``rtype``. + """ + resources = {rtype: list(ids)} if rtype else {} + resource_grains = ( + self._collect_resource_grains({rtype: ids}) if rtype and ids else {} + ) + load = { + "cmd": "_register_resources", + "id": self.opts["id"], + "resources": resources, + "resource_grains": resource_grains, + "chunk_rtype": rtype, + "chunk_index": index, + "chunk_final": final, + "tok": self.tok, + } try: - # Build the load inside the guard as well: this method is - # best-effort, so a problem assembling the request should be - # reported the same way a failure to send it is, rather than - # escaping into pillar_refresh and aborting the rest of it. - load = { - "cmd": "_register_resources", - "id": self.opts["id"], - "resources": resources, - "resource_grains": resource_grains, - "tok": self.tok, - } await self._send_req_async_main(load, timeout=self._return_retry_timer()) - log.debug("Registered resources with master: %s", list(resources.keys())) + log.debug( + "Registered resource chunk type=%s index=%d final=%s count=%d", + rtype, + index, + final, + len(resources.get(rtype, []) if rtype else []), + ) except Exception as err: # pylint: disable=broad-except - log.warning("Unable to register resources with master: %s", err) + log.warning( + "Unable to register resource chunk (type=%s index=%d): %s", + rtype, + index, + err, + ) def module_refresh(self, force_refresh=False, notify=False): """ @@ -4907,11 +4978,11 @@ async def pillar_refresh(self, force_refresh=False, clean_cache=False): # Proxy-scoped so a regular minion pays no extra rebuild. self.module_refresh(force_refresh) - # Re-discover resources now that pillar has changed. Must - # happen *after* opts["pillar"] is updated so that - # _discover_resources sees the new resource declarations (or - # their absence when a type is removed from the pillar). - self.opts["resources"] = self._discover_resources() + # Re-register resources with the master. Inventory lives + # in minion opts (``resource_types`` + per-type keys), not + # in pillar -- a pillar refresh does not change the ID + # set, but we still re-stream so transient loader state + # (e.g. newly synced resource modules) is picked up. await self._register_resources_with_master() finally: # See ``_post_master_init`` for why aclose is preferred @@ -5158,7 +5229,6 @@ async def handle_event(self, package): elif tag.startswith("matchers_refresh"): _minion.matchers_refresh() elif tag.startswith("resource_refresh"): - _minion.opts["resources"] = _minion._discover_resources() _minion._spawn_background(_minion._register_resources_with_master()) elif tag.startswith("manage_schedule"): _minion.manage_schedule(tag, data) @@ -5972,10 +6042,13 @@ def _is_pure_resource_target(self, load): and tgt != self.opts.get("id") and not any(c in tgt for c in ("*", "?", "[")) ): - resources = self.opts.get("resources", {}) - for rids in resources.values(): - if tgt in rids: - return True + # Scan each enabled type's discover() stream for a bare-id + # match. Early-exit on hit keeps this bounded in the common + # case; the generator is torn down as soon as we return. + for rtype in salt.utils.resources.enabled_resource_types(self.opts): + for rid in self._iter_resource_ids(rtype): + if tgt == rid: + return True return False # Functions that are internal Salt plumbing and should never be dispatched @@ -6132,29 +6205,35 @@ def _resource_matches_compound(self, tgt, srn, gdict, resources): def _resolve_resource_targets(self, load): """ - Return the list of per-resource dicts ``{"id": ..., "type": ...}`` that - the target expression matches against ``opts["resources"]``. - - For wildcard glob targets (e.g. ``salt '*'``), returns all managed - resources so that the command also runs against resources. - For compound T@ targets, returns only the matched resources. - For list targets, returns resources whose bare id appears in the list. - For an exact glob with no wildcards, returns a single resource if ``tgt`` - is a bare id managed by this minion (``salt ``). - For specific-name glob targets that are not resource ids (e.g. - ``salt 'minion'``), grain, pillar, or compound expressions with no T@ - terms, returns an empty list — the operator is targeting the minion - itself, not its resources. - Internal/plumbing functions (see ``_NO_RESOURCE_FUNS``) are never - dispatched to resources. - """ - resources = self.opts.get("resources", {}) - if not resources: + Return the list of per-resource dicts ``{"id": ..., "type": ...}`` + that the target expression matches against the resources this + minion manages. + + Resource IDs are streamed per enabled type via + :meth:`_iter_resource_ids` -- no inventory dict is materialized + in ``self.opts``. The resulting target list is bounded by the + matched-resource count, not by the total inventory. + + For wildcard glob targets (e.g. ``salt '*'``), returns all + managed resources so that the command also runs against + resources. For compound T@ targets, returns only the matched + resources. For list targets, returns resources whose bare id + appears in the list. For an exact glob with no wildcards, + returns a single resource if ``tgt`` is a bare id managed by + this minion (``salt ``). For specific-name glob + targets that are not resource ids (e.g. ``salt 'minion'``), + grain, pillar, or compound expressions with no T@ terms, returns + an empty list -- the operator is targeting the minion itself, + not its resources. Internal/plumbing functions (see + ``_NO_RESOURCE_FUNS``) are never dispatched to resources. + """ + enabled_types = salt.utils.resources.enabled_resource_types(self.opts) + if not enabled_types: return [] fun = load.get("fun") - # Multi-fun jobs (``fun`` is a list) bypass resource dispatch — the - # multifun execution path doesn't fold per-resource returns. + # Multi-fun jobs (``fun`` is a list) bypass resource dispatch -- + # the multifun execution path doesn't fold per-resource returns. if not isinstance(fun, str): return [] if fun in self._NO_RESOURCE_FUNS: @@ -6164,29 +6243,26 @@ def _resolve_resource_targets(self, load): tgt_type = load.get("tgt_type", "glob") if tgt_type == "compound": - # Per-resource boolean evaluation of the compound expression. - # Each resource is treated as an independent target whose - # identity (``type``, ``id``) and grain dict drive the truth - # value of every term; ``and`` / ``or`` / ``not`` / parens are - # evaluated using Python's boolean operators after rendering - # each term to ``True``/``False``. - rg = getattr(self, "_resource_grains_cache", None) - if rg is None: - rg = self._collect_resource_grains() - self._resource_grains_cache = rg - # Include every managed resource even if it has no grain entry - # — so plain ``T@type`` / ``T@type:id`` compounds still match - # for resources whose connection module exposes no ``grains``. - all_srns = dict(rg) - for rtype, rids in (resources or {}).items(): - for rid in rids or (): - all_srns.setdefault(f"{rtype}:{rid}", {}) + # Compound terms reference per-resource grains + identity. + # Stream per-type so the peak resident set is bounded by a + # single type's chunk -- NOT by the full inventory. Grain + # dicts for a type are rendered once per dispatch and + # dropped after the per-type scan, so a compound match + # against 200K resources never holds all 200K grain dicts + # resident. + resources_view = { + rtype: list(self._iter_resource_ids(rtype)) for rtype in enabled_types + } targets = [] - for srn, gdict in all_srns.items(): - if self._resource_matches_compound(tgt, srn, gdict, resources): - rtype, _, rid = srn.partition(":") - if rid: + for rtype, rids in resources_view.items(): + rg = self._collect_resource_grains({rtype: rids}) + for rid in rids: + srn = f"{rtype}:{rid}" + gdict = rg.get(srn, {}) + if self._resource_matches_compound(tgt, srn, gdict, resources_view): targets.append({"id": rid, "type": rtype}) + # Drop this type's grain dict before moving to the next. + del rg return targets if tgt_type == "list": @@ -6195,54 +6271,61 @@ def _resolve_resource_targets(self, load): if isinstance(tgt, str) else [str(t).strip() for t in tgt if str(t).strip()] ) + if not tokens: + return [] + token_set = set(tokens) targets = [] - for token in tokens: - for rtype, rids in resources.items(): - if token in rids: - targets.append({"id": token, "type": rtype}) + for rtype in enabled_types: + for rid in self._iter_resource_ids(rtype): + if rid in token_set: + targets.append({"id": rid, "type": rtype}) return targets if tgt_type in ("grain", "grain_pcre"): # Match each managed resource's per-resource grains against - # ``tgt`` (a ``key:value`` expression). Uses the cache populated - # by :meth:`_register_resources_with_master`; refreshes on miss - # so a never-registered minion still resolves its own targets. - rg = getattr(self, "_resource_grains_cache", None) - if rg is None: - rg = self._collect_resource_grains() - self._resource_grains_cache = rg + # ``tgt`` (a ``key:value`` expression). Render grains + # per-type so a 200K-resource minion never holds all grain + # dicts resident -- each type's grain dict is dropped after + # its scan completes. targets = [] - for srn, gdict in rg.items(): - if salt.utils.data.subdict_match( - gdict, - tgt, - delimiter=DEFAULT_TARGET_DELIM, - regex_match=(tgt_type == "grain_pcre"), - ): - rtype, _, rid = srn.partition(":") - if rid: - targets.append({"id": rid, "type": rtype}) + for rtype in enabled_types: + rids = list(self._iter_resource_ids(rtype)) + rg = self._collect_resource_grains({rtype: rids}) + for srn, gdict in rg.items(): + if salt.utils.data.subdict_match( + gdict, + tgt, + delimiter=DEFAULT_TARGET_DELIM, + regex_match=(tgt_type == "grain_pcre"), + ): + _rtype, _, rid = srn.partition(":") + if rid: + targets.append({"id": rid, "type": _rtype}) + del rg return targets # For glob targets, only dispatch to resources when the pattern # contains a wildcard. A bare name like ``salt 'minion' test.ping`` - # targets the minion itself; it should not implicitly run against its - # resources. ``salt '*' test.ping`` or ``salt 'web*' test.ping`` - # opts in to resource dispatch. + # targets the minion itself; it should not implicitly run against + # its resources. if ( tgt_type == "glob" and isinstance(tgt, str) and not any(c in tgt for c in ("*", "?", "[")) ): - for rtype, rids in resources.items(): - if tgt in rids: - return [{"id": tgt, "type": rtype}] + for rtype in enabled_types: + for rid in self._iter_resource_ids(rtype): + if rid == tgt: + return [{"id": tgt, "type": rtype}] return [] - # Wildcard glob — dispatch to all managed resources. + # Wildcard glob -- dispatch to every managed resource. This + # path is inherently O(inventory) in the number of returned + # targets; callers that care about memory should scope their + # target pattern tighter. all_resources = [] - for rtype, rids in resources.items(): - for rid in rids: + for rtype in enabled_types: + for rid in self._iter_resource_ids(rtype): all_resources.append({"id": rid, "type": rtype}) return all_resources diff --git a/salt/resources/dummy/__init__.py b/salt/resources/dummy/__init__.py index 4cf56361343f..e80fb483100f 100644 --- a/salt/resources/dummy/__init__.py +++ b/salt/resources/dummy/__init__.py @@ -2,24 +2,29 @@ Dummy resource module for testing the Salt resource subsystem. This module implements the ``dummy`` resource type. It is the resource -analogue of ``salt.proxy.dummy`` — a self-contained, file-backed -implementation that exercises the full resource lifecycle without requiring -any real managed devices. - -Unlike a proxy module, a resource module is loaded **once per resource type -per minion**. A single instance of this module handles all ``dummy`` -resources managed by the minion. The current resource context is conveyed -via the ``__resource__`` dunder rather than as a function parameter, keeping -the interface consistent with all other Salt module systems. - -Configuration (via Pillar):: - - resources: - dummy: - resource_ids: - - dummy-01 - - dummy-02 - - dummy-03 +analogue of ``salt.proxy.dummy`` -- a self-contained, file-backed +implementation that exercises the full resource lifecycle without +requiring any real managed devices. + +Unlike a proxy module, a resource module is loaded **once per resource +type per minion**. A single instance of this module handles all +``dummy`` resources managed by the minion. The current resource context +is conveyed via the ``__resource__`` dunder rather than as a function +parameter, keeping the interface consistent with all other Salt module +systems. + +Configuration (minion opts, e.g. ``/etc/salt/minion`` or a +``minion.d/*.conf`` drop-in):: + + resource_types: + - dummy + dummy_resource_count: 200000 + +``dummy_resource_count`` controls how many synthetic resources the +minion manages. IDs are generated programmatically as +``dummy-0000001 .. dummy-NNNNNNN`` -- never materialized into +``opts`` or ``pillar``. ``discover()`` is a generator so the register +loop can stream chunks without ever holding the full ID list in memory. """ import copy @@ -30,11 +35,25 @@ import salt.utils.files import salt.utils.msgpack -import salt.utils.resources log = logging.getLogger(__name__) +def _dummy_resource_count(opts): + """Return the configured ``dummy_resource_count`` as a non-negative int. + + Missing / invalid values yield ``0`` (minion manages no dummy + resources). Negative values are clamped to ``0``. + """ + raw = opts.get("dummy_resource_count", 0) + try: + count = int(raw) + except (TypeError, ValueError): + log.warning("dummy_resource_count must be an int; got %r -- treating as 0", raw) + return 0 + return max(count, 0) + + def __virtual__(): """ Always available — no external dependencies required. @@ -76,9 +95,7 @@ def _save_state(opts, resource_id, details): cachefile = os.path.join(opts["cachedir"], f"dummy-resource-{resource_id}.cache") with salt.utils.files.fopen(cachefile, "wb") as pck: pck.write(salt.utils.msgpack.packb(details, use_bin_type=True)) - log.warning( - "Dummy Resource Saved State(%s):\n%s", cachefile, pprint.pformat(details) - ) + log.debug("Dummy Resource Saved State(%s):\n%s", cachefile, pprint.pformat(details)) def _load_state(opts, resource_id): @@ -93,9 +110,7 @@ def _load_state(opts, resource_id): log.exception("Failed to load state: %s", exc, exc_info=True) state = _initial_state(resource_id) _save_state(opts, resource_id, state) - log.warning( - "Dummy Resource Loaded State(%s):\n%s", cachefile, pprint.pformat(state) - ) + log.debug("Dummy Resource Loaded State(%s):\n%s", cachefile, pprint.pformat(state)) return state @@ -119,24 +134,18 @@ def init(opts): """ Initialize the dummy resource type for this minion. - Called once when the resource type is loaded, before any per-resource - operations are performed. Reads the resource type configuration from the - ``dummy`` entry under the pillar subtree selected by ``resource_pillar_key`` - (see :func:`salt.utils.resources.pillar_resources_tree`) and sets up shared type-level - state in ``__context__["dummy_resource"]``. - - :param dict opts: The Salt opts dict. + Called once when the resource type is loaded, before any + per-resource operations are performed. The count of managed dummy + resources is read from the ``dummy_resource_count`` minion opt; the + ID list itself is never materialized here -- callers that need to + iterate IDs use :func:`discover` which streams them on demand. """ - resource_ids = ( - salt.utils.resources.pillar_resources_tree(opts) - .get("dummy", {}) - .get("resource_ids", []) - ) + count = _dummy_resource_count(opts) __context__["dummy_resource"] = { "initialized": True, - "resource_ids": resource_ids, + "resource_count": count, } - log.debug("dummy resource init() called, managing: %s", resource_ids) + log.debug("dummy resource init() called, managing %d resource(s)", count) def initialized(): @@ -154,27 +163,22 @@ def initialized(): def discover(opts): """ - Return the list of resource IDs of type ``dummy`` that this minion - manages. + Yield bare resource IDs of type ``dummy`` that this minion manages. - Called by ``saltutil.refresh_resources`` to populate the master's - Resource Registry. For the dummy module the list of IDs is read from - ``resource_ids`` under the ``dummy`` type in the configured resource pillar - subtree. - - Returns a list of bare resource IDs (not full SRNs) — e.g. - ``["dummy-01", "dummy-02"]``. + This is a **generator**: IDs are produced one at a time so the + minion's chunked register loop can ship fixed-size chunks without + the full inventory ever being resident. The count is driven by + the ``dummy_resource_count`` minion opt; IDs follow the pattern + ``dummy-NNNNNNN`` (zero-padded to 7 digits). :param dict opts: The Salt opts dict. - :rtype: list[str] - """ - resource_ids = ( - salt.utils.resources.pillar_resources_tree(opts) - .get("dummy", {}) - .get("resource_ids", []) - ) - log.debug("dummy resource discover() returning: %s", resource_ids) - return resource_ids + :rtype: Iterator[str] + """ + count = _dummy_resource_count(opts) + prefix = str(opts.get("dummy_resource_id_prefix") or "") + log.debug("dummy resource discover() streaming %d id(s) prefix=%r", count, prefix) + for i in range(1, count + 1): + yield f"{prefix}dummy-{i:07d}" def grains(): diff --git a/salt/resources/ssh/__init__.py b/salt/resources/ssh/__init__.py index 8f6380df32df..5369d79b9d30 100644 --- a/salt/resources/ssh/__init__.py +++ b/salt/resources/ssh/__init__.py @@ -11,21 +11,28 @@ the salt-thin bundle for grain collection (``grains.items``), giving the same complete, accurate grain set that ``salt-ssh`` provides. -Configuration (via Pillar; top-level key defaults to ``resources``, overridable -with minion option ``resource_pillar_key``):: - - resources: - ssh: - hosts: - web-01: - host: 192.168.1.10 - user: root - priv: /etc/salt/ssh_keys/web-01 - web-02: - host: 192.168.1.11 - user: admin - passwd: secretpassword - no_host_keys: true +Configuration (minion opts, e.g. ``/etc/salt/minion`` or a +``minion.d/*.conf`` drop-in):: + + resource_types: + - ssh + ssh: + hosts: + web-01: + host: 192.168.1.10 + user: root + priv: /etc/salt/ssh_keys/web-01 + web-02: + host: 192.168.1.11 + user: admin + passwd: secretpassword + no_host_keys: true + +Pillar is intentionally NOT the source of resource inventory on the +minion -- holding a large hosts dict in ``opts["pillar"]`` violates +the minion memory ceiling. The ``ssh`` opts key is the single +source of truth for host configuration; ``saltutil.refresh_resources`` +re-reads it on demand without a process restart. Per-host connection parameters: @@ -81,7 +88,6 @@ import salt.utils.json import salt.utils.network import salt.utils.path -import salt.utils.resources log = logging.getLogger(__name__) @@ -302,16 +308,16 @@ def init(opts): Initialize the ``ssh`` resource type for this minion. Called once when the resource type is loaded, before any per-resource - operations are dispatched. Reads host configs from the ``ssh`` entry under - the pillar subtree selected by ``resource_pillar_key`` (see - :func:`salt.utils.resources.pillar_resources_tree`), caches them in - ``__context__["ssh_resource"]``, and pre-resolves the SSH binary version - so that :func:`_shell_opts` never has to run a subprocess during a job. + operations are dispatched. Reads host configs from the top-level + ``ssh`` opts key (same opts-driven pattern as the other resource + modules), caches them in ``__context__["ssh_resource"]``, and + pre-resolves the SSH binary version so that :func:`_shell_opts` + never has to run a subprocess during a job. :param dict opts: The Salt opts dict. """ - resource_cfg = salt.utils.resources.pillar_resources_tree(opts).get("ssh", {}) - hosts = resource_cfg.get("hosts", {}) + resource_cfg = opts.get("ssh", {}) if isinstance(opts.get("ssh"), dict) else {} + hosts = resource_cfg.get("hosts", {}) if isinstance(resource_cfg, dict) else {} __context__[CONTEXT_KEY] = { # pylint: disable=undefined-variable "initialized": True, "hosts": hosts, @@ -364,22 +370,29 @@ def initialized(): def discover(opts): """ - Return the list of SSH resource IDs managed by this minion. + Yield SSH resource IDs managed by this minion. - The list is the set of keys under ``hosts`` for the ``ssh`` type under the - configured resource pillar subtree. Adding or removing a - host from that Pillar key and running ``saltutil.refresh_resources`` - updates the Master's Resource Registry without any process restart. + The list is the set of keys under ``hosts`` in the top-level ``ssh`` + opts dict. Adding or removing a host from that key and running + ``saltutil.refresh_resources`` updates the Master's Resource + Registry without any process restart. + + This is a generator so the chunked register loop in + :meth:`salt.minion.Minion._register_resources_with_master` can + stream fixed-size chunks without materializing the full host + list -- matches the dummy module's streaming contract. :param dict opts: The Salt opts dict. - :rtype: list[str] + :rtype: Iterator[str] """ - hosts = ( - salt.utils.resources.pillar_resources_tree(opts).get("ssh", {}).get("hosts", {}) - ) - resource_ids = list(hosts) - log.debug("ssh resource discover() returning: %s", resource_ids) - return resource_ids + cfg = opts.get("ssh") + hosts = {} + if isinstance(cfg, dict): + cfg_hosts = cfg.get("hosts") + if isinstance(cfg_hosts, dict): + hosts = cfg_hosts + log.debug("ssh resource discover() yielding %d id(s)", len(hosts)) + yield from hosts def grains(): diff --git a/salt/utils/grain_target_cache.py b/salt/utils/grain_target_cache.py new file mode 100644 index 000000000000..8a41cdb08156 --- /dev/null +++ b/salt/utils/grain_target_cache.py @@ -0,0 +1,276 @@ +""" +In-process sorted-key cache for ``resource_id:*`` grain-target +predicates. + +Companion to :mod:`salt.utils.minions` — specifically the +``_augment_grain_match_with_resource_grains`` grain-target expansion path +that walks the ``resource_grains`` cache bank. The literal-id fast path +(Mission 1) handles ``resource_id:`` via an O(T) hash probe; +this module handles the next tier -- globs with a non-empty fixed prefix +such as ``resource_id:web-*`` or ``resource_id:a-dummy-00*`` -- by +binary-searching a sorted copy of the bank's keys so we iterate only the +prefix range instead of linearly scanning ~400K entries. + +Scope +----- +* Query-only. Reads from the existing cache bank; never writes. +* Per-process in-memory index. No disk file, no mmap, no persistence. +* Keyed by ``(cachedir, bank)`` so a process with multiple cache dirs + (test harness, multi-master-in-process) keeps them separate. + +Invalidation +------------ +A cheap per-bank "generation" signal is sampled on every query. If the +signal matches the one recorded at build time the cached sorted list is +reused verbatim; otherwise it is rebuilt. Signals (best available for +each driver): + +* ``mmap_cache``: the 8-byte ``occupied_count`` header field at offset 1 + of ``/.mmap_cache.idx``. The master bumps this on every + ``store`` / delete, so any write to the bank advances the signal + without us having to coordinate with the writer. +* ``localfs`` (and any other dir-backed driver): ``st_mtime_ns`` of the + bank directory. Directory mtime advances whenever a ``.p`` file is + added, removed, or renamed. +* Anything else / signal unavailable: fall back to an "always rebuild" + tag so correctness is preserved at the cost of the fast path's + speedup. + +Thread safety +------------- +The module exposes a singleton :class:`GrainTargetCache` guarded by a +single ``RLock``. The hot path (same generation) takes the lock, returns +the cached list (an immutable-by-convention ``list`` reference), and +releases. Rebuilds hold the lock. A caller that holds the returned list +may keep iterating it even if another thread subsequently rebuilds -- +the old list stays valid because we always replace, never mutate. +""" + +import bisect +import logging +import os +import struct +import threading + +import salt.utils.files + +log = logging.getLogger(__name__) + +# mmap_cache header: offset 1, uint64 little-endian = occupied_count. +# Mirrors the constants in salt.utils.mmap_cache but duplicated here so +# we do not force-import the mmap_cache module (and its xxhash +# dependency) in every CkMinions process just to read 8 header bytes. +_MMAP_INDEX_FILENAME = ".mmap_cache.idx" +_MMAP_HDR_OCCUPIED_OFF = 1 +_MMAP_HDR_OCCUPIED_FMT = ":`` where ``rid`` + is a bare id composed of ``[A-Za-z0-9._-]`` -- bracket classes are + extremely rare and the fall-through to the linear scan handles them. + + Returns an empty string (not ``None``) when the pattern has no fixed + prefix. ``None`` input returns ``None``. + """ + if not isinstance(glob_pattern, str): + return None + prefix_chars = [] + for ch in glob_pattern: + if ch in ("*", "?", "["): + break + prefix_chars.append(ch) + return "".join(prefix_chars) + + +def prefix_range_candidates(sorted_keys, id_prefix): + """ + Yield the keys from ``sorted_keys`` that start with ``id_prefix``. + + ``sorted_keys`` must be sorted by Python's default string ordering + (``sorted(...)`` with no key). ``id_prefix`` may be empty, in which + case the full list is yielded (callers normally short-circuit + without calling us in that case). + + Uses ``bisect_left`` for the lower bound and walks forward until a + key no longer starts with the prefix -- O(log N + matches). + """ + if not sorted_keys: + return + if not id_prefix: + yield from sorted_keys + return + lo = bisect.bisect_left(sorted_keys, id_prefix) + n = len(sorted_keys) + i = lo + while i < n: + k = sorted_keys[i] + if not k.startswith(id_prefix): + break + yield k + i += 1 + + +def _mmap_index_path(cachedir, bank): + """Return the expected ``.mmap_cache.idx`` path for ``bank``.""" + # bank may contain path separators (nested banks); ``os.path.join`` + # with ``os.path.normpath`` mirrors what the mmap_cache driver does + # internally so we look at the same file. + return os.path.join(cachedir, os.path.normpath(bank), _MMAP_INDEX_FILENAME) + + +def _read_mmap_generation(cachedir, bank): + """ + Return the current mmap_cache ``occupied_count`` for *bank*, or + ``None`` if the index file is missing / too small / unreadable. + + Reads exactly 8 bytes at offset 1 -- O(1). + """ + path = _mmap_index_path(cachedir, bank) + try: + with salt.utils.files.fopen(path, "rb") as fh: + fh.seek(_MMAP_HDR_OCCUPIED_OFF) + buf = fh.read(_MMAP_HDR_OCCUPIED_SIZE) + except (FileNotFoundError, PermissionError, OSError): + return None + if len(buf) != _MMAP_HDR_OCCUPIED_SIZE: + return None + try: + return struct.unpack(_MMAP_HDR_OCCUPIED_FMT, buf)[0] + except struct.error: + return None + + +def _read_dir_generation(cachedir, bank): + """ + Return the bank-directory's ``st_mtime_ns``, or ``None`` if the + directory does not exist / is not accessible. + """ + path = os.path.join(cachedir, os.path.normpath(bank)) + try: + st = os.stat(path) + except (FileNotFoundError, PermissionError, OSError): + return None + return st.st_mtime_ns + + +def _bank_generation(cache, bank): + """ + Return a cheap O(1) signal that changes when *bank* is written to. + + Tries the mmap-cache ``occupied_count`` header first (available iff + the bank directory contains ``.mmap_cache.idx``); falls back to the + bank directory's ``st_mtime_ns``; returns :data:`_GEN_UNKNOWN` when + neither can be derived (correctness-preserving "always rebuild"). + """ + cachedir = getattr(cache, "cachedir", None) + if not cachedir: + return _GEN_UNKNOWN + mmap_gen = _read_mmap_generation(cachedir, bank) + if mmap_gen is not None: + return ("mmap", mmap_gen) + dir_gen = _read_dir_generation(cachedir, bank) + if dir_gen is not None: + return ("dir", dir_gen) + return _GEN_UNKNOWN + + +class GrainTargetCache: + """ + Per-process cache of ``{(cachedir, bank): (generation, sorted_keys)}``. + + Not instantiated directly by callers -- use :func:`get` for the + module-level singleton. A dedicated instance is still useful for + tests that need isolation from the module-level state. + """ + + def __init__(self): + self._entries = {} + self._lock = threading.RLock() + + def _cache_key(self, cache, bank): + cachedir = getattr(cache, "cachedir", None) or "" + return (cachedir, bank) + + def invalidate(self, cache=None, bank=None): + """ + Drop cached entries. With no arguments drops everything; with + ``cache`` and ``bank`` drops only that bank. + """ + with self._lock: + if cache is None and bank is None: + self._entries.clear() + return + key = self._cache_key(cache, bank) + self._entries.pop(key, None) + + def get_sorted_keys(self, cache, bank): + """ + Return the current sorted list of keys in ``bank``, rebuilding + if the bank-generation signal has advanced since the last build. + + Returns a list -- possibly empty. Callers must treat the list as + immutable; the cache keeps the same object on subsequent hits + and may hand it to concurrent readers. + """ + key = self._cache_key(cache, bank) + with self._lock: + current_gen = _bank_generation(cache, bank) + entry = self._entries.get(key) + if entry is not None: + cached_gen, cached_keys = entry + # _GEN_UNKNOWN never compares equal (sentinel object) so + # banks with no cheap signal always rebuild -- correct + # if slow. + if cached_gen is not _GEN_UNKNOWN and cached_gen == current_gen: + return cached_keys + try: + raw = cache.list(bank) or [] + except Exception: # pylint: disable=broad-except + log.debug( + "grain_target_cache: cache.list(%r) failed; " + "returning empty key list", + bank, + exc_info=True, + ) + raw = [] + # Defensive: cache.list may return an iterable (generator), + # tuple, set, or non-string entries. Stringify + sort. + keys = sorted(str(k) for k in raw) + self._entries[key] = (current_gen, keys) + return keys + + +# Module-level singleton. Tests that need isolation can instantiate +# their own :class:`GrainTargetCache`; the production hot path uses this +# one so a single CkMinions process does not pay the enumeration cost +# per-instance. +_default = GrainTargetCache() + + +def get(): + """Return the module-level :class:`GrainTargetCache` singleton.""" + return _default + + +def get_sorted_keys(cache, bank): + """Shortcut for ``get().get_sorted_keys(cache, bank)``.""" + return _default.get_sorted_keys(cache, bank) + + +def invalidate(cache=None, bank=None): + """Shortcut for ``get().invalidate(cache, bank)``.""" + _default.invalidate(cache=cache, bank=bank) diff --git a/salt/utils/minions.py b/salt/utils/minions.py index 5e69feea04ce..4e838c918020 100644 --- a/salt/utils/minions.py +++ b/salt/utils/minions.py @@ -14,6 +14,8 @@ import salt.transport import salt.utils.data import salt.utils.files +import salt.utils.grain_target_cache +import salt.utils.mmap_trie_index import salt.utils.network import salt.utils.resource_registry import salt.utils.resources @@ -398,6 +400,180 @@ def update_resource_index(opts, minion_id, resources): return (0, 0) +def update_resource_index_chunk( + opts, minion_id, resources, chunk_final, chunk_scope=None, chunk_index=None +): + """ + Chunk-aware wrapper around + :meth:`ResourceRegistry.register_minion_chunk`. + + Non-final chunks ``put_many`` their SRNs with no delete pass; only the + terminal chunk prunes stale entries against the window's cumulative + SRN set. This is the correctness fix for multi-chunk registers -- + without it, each chunk's receipt would diff against the previous + chunk's just-written entries and prune them, leaving only the last + chunk persisted. + + :param dict opts: Salt opts (needs ``cachedir``). + :param str minion_id: The reporting minion. + :param dict resources: ``{resource_type: [resource_id, ...]}`` -- this + chunk's slice. + :param bool chunk_final: True on the terminal chunk of a register + window (triggers the stale-SRN prune). + :param str chunk_scope: Which type(s) this register window covers. + ``None`` or ``"__all__"`` means the window replaces the minion's + full inventory (legacy single-shot). A type name scopes the + prune to that type only. + :param int chunk_index: Monotonically-increasing index assigned by + the minion-side sender (0-based within the window). Used by + the registry to detect and defer out-of-order terminal prunes + on peer masters. ``None`` keeps pre-chunk semantics for legacy + callers. + :returns: ``(n_put, n_deleted)`` for logging. ``n_deleted`` is + always zero on non-final chunks and on terminal chunks whose + prune was deferred by a chunk-index gap. + """ + try: + registry = salt.utils.resource_registry.get_registry(opts) + except Exception: # pylint: disable=broad-except + log.error( + "Failed to open resource registry while registering %s; " + "resource targeting for this minion will be unavailable.", + minion_id, + exc_info=True, + ) + return (0, 0) + try: + return registry.register_minion_chunk( + minion_id, + resources or {}, + chunk_final, + chunk_scope, + chunk_index=chunk_index, + ) + except Exception: # pylint: disable=broad-except + log.error( + "Failed to register resource chunk for minion '%s' (scope=%s final=%s)", + minion_id, + chunk_scope, + chunk_final, + exc_info=True, + ) + return (0, 0) + + +def _is_literal_glob(pattern): + """ + Return ``True`` iff ``pattern`` has no :mod:`fnmatch` wildcards. + + A "literal" pattern contains none of ``*``, ``?``, or ``[`` -- so + ``fnmatch.fnmatchcase(x, pattern)`` reduces to ``x == pattern``. + Backslash escapes are NOT treated as special (``fnmatch`` does not + honour them either on any platform), matching the existing fast-path + wildcard check in :meth:`CkMinions._check_glob_minions`. + """ + if not isinstance(pattern, str) or not pattern: + return False + return not any(c in pattern for c in ("*", "?", "[")) + + +def _is_resource_id_literal_predicate(expr, delimiter): + """ + Return ``True`` iff ``expr`` is a flat ``resource_id:`` + predicate suitable for the exact-key hash-probe fast path. + + Mirrors the shape checks in :func:`_resource_id_candidates` (key is + exactly ``resource_id``, non-empty pattern, no nested delimiter) and + adds the literal-glob check so ``resource_id:a-*`` and friends fall + through to the existing bank-scan fast path. + """ + if not isinstance(expr, str): + return False + key, sep, pattern = expr.partition(delimiter) + if not sep or key != "resource_id" or not pattern: + return False + if delimiter in pattern: + return False + return _is_literal_glob(pattern) + + +def _resource_id_prefix_predicate(expr, delimiter, regex_match): + """ + Return ``(id_prefix, full_pattern)`` iff ``expr`` is a flat + ``resource_id:`` predicate with a non-empty fixed prefix. + + Mirrors the shape checks of :func:`_resource_id_candidates` (same + key / no nested delimiter / non-empty pattern) and adds the + prefix-non-empty / non-literal requirements so the caller can + dispatch to the sorted-key fast path. Returns ``None`` when the + predicate shape doesn't apply, when the glob is literal (the + Mission 1 hash-probe path already handled it), when regex matching + is in effect (prefix analysis of arbitrary regex is not worth the + complexity), or when the glob has no fixed prefix (e.g. starts with + ``*`` or ``[``). + """ + if regex_match or not isinstance(expr, str): + return None + key, sep, pattern = expr.partition(delimiter) + if not sep or key != "resource_id" or not pattern: + return None + if delimiter in pattern: + return None + # Literal globs are already served by the Mission 1 hash probe; we + # only want to handle the "prefix + wildcard tail" shape. + if _is_literal_glob(pattern): + return None + prefix = salt.utils.grain_target_cache.fixed_prefix_of(pattern) + if not prefix: + return None + return prefix, pattern + + +def _resource_id_candidates(sr_keys, expr, delimiter, regex_match): + """ + Narrow ``sr_keys`` to the subset whose id portion matches a + top-level ``resource_id:`` predicate. + + The ``resource_grains`` bank stores one entry per salt resource, + keyed by ``:``. By convention the ``resource_id`` grain + equals the id portion of that key, so a ``resource_id:`` + target can be narrowed by filename pattern alone — without + pickle-loading every grain dict. Callers still fetch + confirm + via ``subdict_match`` so the convention does not have to hold + for correctness; this helper only trims the fetch set. + + Returns ``None`` when the fast path does not apply (nested + predicate, different key, empty pattern), signalling the caller + to fall back to a full-bank scan. + """ + if not isinstance(expr, str): + return None + key, sep, pattern = expr.partition(delimiter) + if not sep or key != "resource_id" or not pattern: + return None + if delimiter in pattern: + # Nested predicate (e.g. ``resource_id:sub:value``) — we only + # know how to short-circuit the flat top-level case. + return None + if regex_match: + try: + rx = re.compile(pattern) + except re.error: + return None + match = rx.search + else: + + def match(rid, _pat=pattern): + return fnmatch.fnmatchcase(rid, _pat) + + out = [] + for sr_key in sr_keys: + _rtype, sep2, rid = sr_key.partition(":") + if sep2 and rid and match(rid): + out.append(sr_key) + return out + + class CkMinions: """ Used to check what minions should respond from a target @@ -619,6 +795,42 @@ def list_cached_minions(): if not cminions: return {"minions": [], "missing": []} minions = set(minions) + # Trie fast path for the per-minion ``grains`` bank. Opt-in + # via ``opts["enable_grain_trie_index"]``. The trie covers + # every scalar leaf of every cached minion grain dict, so + # arbitrary ``:`` and nested predicates are + # served without the per-minion pickle-fetch loop below. + # ``exact_match=True`` keeps the old behaviour (the trie's + # literal path already short-circuits on equality so there + # is no separate exact-match dispatch). Returns ``None`` + # when the trie cannot authoritatively answer (cap hit, + # missing build, shape mismatch); we then fall through to + # the legacy loop. + if search_type == "grains" and self.opts.get("enable_grain_trie_index"): + try: + trie_hits = salt.utils.mmap_trie_index.dispatch_predicate( + self.cache, + "grains", + self.opts, + expr, + delimiter, + regex_match, + ) + except Exception: # pylint: disable=broad-except + log.debug( + "mmap_trie_index: grains dispatch failed for expr=%r; " + "falling through to slow path", + expr, + exc_info=True, + ) + trie_hits = None + if trie_hits is not None: + # Intersect with the starting set so greedy ∩ PKI + # and non-greedy ∩ cached-list semantics are both + # preserved; the trie only knows which minions + # have cached grains, not which are accepted. + filtered = [m for m in trie_hits if m in minions] + return {"minions": filtered, "missing": []} # Track which accepted minions have cache data we can evaluate. # Any accepted minion not present in ``cminions`` has no cache # entry and must be excluded; otherwise grain/pillar targeting @@ -694,17 +906,145 @@ def _augment_grain_match_with_resource_grains( if not self.opts.get("minion_data_cache", False): return bank = salt.utils.resource_registry.RESOURCE_GRAINS_BANK + existing = result.get("minions", []) + seen = set(existing) + # Normalise to list up-front so downstream consumers get a stable + # shape regardless of which branch returns (``_check_cache_minions`` + # may hand us a ``set``). + if not isinstance(existing, list): + result["minions"] = list(existing) + + # Literal-exact fast path for ``resource_id:`` targets + # (no wildcards, no delimiter nesting, non-regex). The master + # enforces ``bank-key == ":"`` at register time, so an + # ``O(T)`` hash probe across the configured resource types is the + # authoritative answer -- no need to list the bank (which is the + # ~O(bank-size) step we are avoiding). + if not regex_match and _is_resource_id_literal_predicate(expr, delimiter): + _, _, literal = expr.partition(delimiter) + try: + rtypes = salt.utils.resources.enabled_resource_types(self.opts) + except Exception: # pylint: disable=broad-except + rtypes = [] + for rtype in rtypes: + try: + hit = self.cache.contains(bank, f"{rtype}:{literal}") + except Exception: # pylint: disable=broad-except + continue + if hit: + if literal not in seen: + result["minions"].append(literal) + seen.add(literal) + # The master enforces id-uniqueness per bank key + # ``:``, but the same bare id *may* exist + # under multiple types; either way we only add it once + # and can return. + return + return + + # Sorted-key prefix fast path for ``resource_id:*`` style + # predicates. Avoids the O(bank-size) ``cache.list()`` + fnmatch + # scan below by binary-searching the sorted bank keys and walking + # only the prefix range. The in-process cache is keyed by + # ``(cachedir, bank)`` and invalidated via a cheap O(1) + # generation signal (mmap_cache occupied-count header or bank + # directory mtime). + prefix_info = _resource_id_prefix_predicate(expr, delimiter, regex_match) + if prefix_info is not None: + id_prefix, pattern = prefix_info + try: + sorted_keys = salt.utils.grain_target_cache.get_sorted_keys( + self.cache, bank + ) + except Exception: # pylint: disable=broad-except + log.debug( + "resource_id prefix fast-path: sorted-key cache fetch failed " + "for bank=%r; falling through to linear scan", + bank, + exc_info=True, + ) + sorted_keys = None + if sorted_keys is not None: + # Bank keys are ``:``; we match against the + # id portion. ``:`` segments sort lexically, so a + # single prefix search across the full sorted-keys list + # would miss matches whose rtype differs. Instead we + # intersect per rtype so each binary-search range only + # covers one type-scoped stride. For a bank with K rtypes + # and N keys total, this is O(K * log N + matches). + try: + rtypes = salt.utils.resources.enabled_resource_types(self.opts) + except Exception: # pylint: disable=broad-except + rtypes = [] + if rtypes: + for rtype in rtypes: + scoped = f"{rtype}:{id_prefix}" + for ( + sr_key + ) in salt.utils.grain_target_cache.prefix_range_candidates( + sorted_keys, scoped + ): + _rtype, sep2, rid = sr_key.partition(":") + if not sep2 or not rid: + continue + if not fnmatch.fnmatchcase(rid, pattern): + continue + if rid not in seen: + result["minions"].append(rid) + seen.add(rid) + return + # No enabled rtypes derivable: fall through to the + # linear-scan path below so correctness is preserved. + try: srns = list(self.cache.list(bank) or []) except Exception: # pylint: disable=broad-except return if not srns: return - existing = result.get("minions", []) - seen = set(existing) - # Normalise to list; preserve any existing order. - if not isinstance(existing, list): - result["minions"] = list(existing) + # Fast path: ``resource_id:`` predicates only need to look + # at the id portion of each bank key (``:``). The master + # enforces ``gdict["resource_id"] == `` at register time (see + # ``salt/master.py:__register_resources_sync``), so a bank-key glob + # match is the full match -- no fetch / ``subdict_match`` needed. + candidates = _resource_id_candidates(srns, expr, delimiter, regex_match) + if candidates is not None: + for srn in candidates: + _rtype, _, rid = srn.partition(":") + if rid and rid not in seen: + result["minions"].append(rid) + seen.add(rid) + return + + # Persistent-trie fast path. Opt-in via + # ``opts["enable_grain_trie_index"]``. The trie covers every + # scalar leaf across every grain dict in the bank, so nested + # predicates (``dns:nameservers:127.0.0.11``) and arbitrary + # grain keys are auto-indexed. Returns ``None`` when the trie + # cannot authoritatively serve the predicate (cap hit, missing + # build, shape mismatch) -- we then fall through to the + # per-key fetch loop below. + if self.opts.get("enable_grain_trie_index"): + try: + trie_hits = salt.utils.mmap_trie_index.dispatch_predicate( + self.cache, bank, self.opts, expr, delimiter, regex_match + ) + except Exception: # pylint: disable=broad-except + log.debug( + "mmap_trie_index: dispatch failed for expr=%r; " + "falling through to slow path", + expr, + exc_info=True, + ) + trie_hits = None + if trie_hits is not None: + for sr_key in trie_hits: + _rtype, _, rid = sr_key.partition(":") + if rid and rid not in seen: + result["minions"].append(rid) + seen.add(rid) + return + for srn in srns: try: gdict = self.cache.fetch(bank, srn) diff --git a/salt/utils/mmap_trie_index.py b/salt/utils/mmap_trie_index.py new file mode 100644 index 000000000000..5330592efffc --- /dev/null +++ b/salt/utils/mmap_trie_index.py @@ -0,0 +1,1637 @@ +""" +Persistent mmap-backed grain trie index for resource-registry +grain-target queries. + +Persists an exhaustive ``(path_tuple, scalar_value) -> [sr_key, ...]`` +mapping on disk using :class:`salt.utils.mmap_cache.MmapCache`, so: + +* **Nested paths are auto-covered.** ``dns:nameservers:127.0.0.11`` or + any other arbitrarily-deep ``:::`` predicate is served + by splitting the expression on the delimiter and probing every valid + (path, value) partition. +* **Process RAM is bounded.** The builder reads one grain dict at a + time and flushes a bounded accumulator (default 1024 entries) to the + mmap trie; the live query only materialises the specific sr_key list + for a hit. + +Scope +----- +* Opt-in via ``opts["enable_grain_trie_index"]``. When off, the + resource-grain dispatch falls back to the legacy per-key + fetch loop. +* Per-bank. Each bank gets its own trie-cache files under + ``//.trie/`` -- never mixed with the primary bank's + mmap files. +* Build is scheduled lazily: queries rebuild when the bank-generation + signal (see :mod:`salt.utils.grain_target_cache`) advances past the + version persisted in ``trie.meta``. + +Key / value schema +------------------ +Every scalar leaf under a grain dict yields exactly one trie entry:: + + key = b"\\x1f" + value = msgpack.packb([sr_key_1, sr_key_2, ...]) + +Separator choice: ``\\x1e`` (ASCII RS) and ``\\x1f`` (ASCII US) are +ASCII control bytes that cannot appear in grain key identifiers +(Python-ish names) nor in the stringified scalar values our callers +emit. **Critically**, neither is NUL: the backing +:class:`MmapCache` uses ``\\x00`` as its key-field padding sentinel +and truncates reads at the first null. Embedding ``\\x00`` in a trie +key would cause silent slot-identity collisions during probing. The +separators are also both below ``0x20`` so they cannot be valid +UTF-8 continuation bytes, keeping the encoding injective across +every grain-value encoding we might encounter. + +High-cardinality buckets (chunked storage) +------------------------------------------ +When a single ``(path, value)`` would accumulate more than +``opts["grain_trie_max_list_size"]`` sr_keys we switch the bucket +layout from "one msgpack list under the natural key" to a **chunked** +layout: + +* The natural trie key now stores a *chunk marker*:: + + msgpack.packb({"_chunks": N, "_total": T}) + + where ``N`` is the number of chunks on disk and ``T`` is the total + sr_key count across those chunks. + +* Chunk ``i`` lives under a derived key:: + + + b"\\x1d" + f"{i:08X}".encode("ascii") + + i.e. a fixed 8-character uppercase-hex rendering of ``i``. Hex + (rather than a raw ``to_bytes`` of the integer) keeps the key free + of NUL bytes, which is necessary because :class:`MmapCache` uses + NUL as the key-field padding sentinel and truncates reads at the + first null. A little-endian uint32 of ``0`` is four NUL bytes -- + that would collapse chunk-0 and all subsequent chunks onto the + same slot. 8 hex characters give 32 bits of index space (``2**32`` + chunks), well beyond any plausible fleet. + + Each chunk's value is a plain msgpack list of sr_keys -- exactly the + same shape as a small non-chunked bucket. + +Separator rationale: ``\\x1d`` (ASCII GS, Group Separator) is reserved +for the chunk-key suffix. It is distinct from the path separators +``\\x1e`` / ``\\x1f`` chosen in Mission 4, is not NUL (so MmapCache's +NUL-pad truncation does not bite), and is not a UTF-8 continuation +byte. A grain scalar value containing ``\\x1d`` would collide with a +chunk-key suffix of a shorter natural key -- extremely unlikely in +practice given the grain-value shapes we encounter, but called out +here as a documented limitation. + +Amortised merge cost: once a bucket is chunked, flushes within a +single rebuild only mutate the LAST chunk until it fills, so cumulative +merge work for a bucket growing to size ``M`` is ``O(M)`` linear, not +``O(M^2 / batch_size)`` as it was under the "re-read the full list every +flush" approach. + +Dict-vs-list unambiguity: a real sr_key leaf is always a msgpack ARRAY +while the chunk marker is a msgpack MAP, so the entry decoder can +dispatch on the leading byte. + +Legacy ``_capped`` sentinel +--------------------------- +Mission 4 (and the first iteration of this module) used a different +sentinel:: + + msgpack.packb({"_capped": True, "count": }) + +to indicate a bucket whose sr_keys had been thrown away past +``max_list_size``. That sentinel is no longer written. For back-compat +with on-disk tries built by an older version we still *recognise* it +in :func:`_unpack_entry`: callers receiving a ``("cap", ...)`` decode +treat it as "data lost, trigger slow path once", and the next rebuild +re-materialises the bucket in chunked form. + +Atomic rebuild +-------------- +``rebuild()`` builds into a fresh :class:`MmapCache` at +``/trie.idx.new`` (plus ``.new.heap`` / ``.new.roster``), +then ``os.replace()`` swaps each file into its live name. The meta +file is swapped last. Concurrent readers see either the pre-rebuild +state or the post-rebuild state -- never a torn intermediate. + +Invalidation +------------ +Reuses :func:`salt.utils.grain_target_cache._bank_generation`. The +builder reads the generation BEFORE it starts the pass, writes it to +``trie.meta`` after the swap. A subsequent reader whose observed +generation matches the meta skips the rebuild. +""" + +import contextlib +import fnmatch +import logging +import os +import re +import tempfile +import threading +import time + +import msgpack + +import salt.utils.files +import salt.utils.grain_target_cache +import salt.utils.mmap_cache + +log = logging.getLogger(__name__) + +# Byte sentinels used inside the mmap key. Chosen because: +# * They never appear in grain path segments (ascii-identifier keys) +# or typical stringified scalar values. +# * Neither is NUL (``\x00``). ``MmapCache`` uses NUL as the +# key-field padding sentinel and truncates reads at the first +# ``\x00``; embedding NUL in trie keys would cause slot +# comparison to never match on read-back. +_PATH_SEP = b"\x1e" # ASCII RS, separates nested path segments +_PATH_VALUE_SEP = b"\x1f" # ASCII US, separates path from value +# ASCII GS, separates the natural bucket key from a chunk index when a +# bucket's sr_key list exceeds ``max_list_size`` and we spill it across +# multiple chunk entries. Reserved: a grain value containing ``\x1d`` +# would collide with a chunk-key suffix of a shorter natural key. The +# collision is theoretical given the scalar shapes grain-flatten emits +# but is called out for forward-compat. +_CHUNK_SEP = b"\x1d" + +# Default knobs mirrored from salt.config so a caller can instantiate +# an index without pulling the full opts dict through. +DEFAULT_MAX_LIST_SIZE = 100000 +DEFAULT_BATCH_SIZE = 1024 +# Trie files sit in a hidden sibling directory of the bank so a bank +# ``flush`` never accidentally stomps the trie (the primary bank only +# scans its own key hash-table). +_TRIE_DIRNAME = ".trie" +_TRIE_INDEX_FILENAME = "trie.idx" +_TRIE_META_FILENAME = "trie.meta" + +# MmapCache sizing for the trie. The trie stores up to +# ``num_scalar_leaves(all_grain_dicts)`` entries. A fleet of 1M SRs x +# 10 grain leaves = 10M distinct keys worst case. We size for ~2x +# headroom (20M slots) by default, which the operator can override +# with ``grain_trie_mmap_size``. +_DEFAULT_MMAP_SIZE = 2_000_000 +# Key size: path_segments-joined + \x1f + value. Grain keys are +# typically short (``dns``, ``nameservers``, etc.) so 128 bytes comfortably +# covers 3-level nesting + reasonable scalar values. Longer keys get +# truncated by MmapCache's slot_size enforcement -- documented in the +# class docstring. +_DEFAULT_KEY_SIZE = 128 +# slot_size must be >= _FIXED_OVERHEAD + key_size = 21 + 128 = 149. +# Round up to a cache-line multiple for struct alignment. +_DEFAULT_SLOT_SIZE = 160 + +# Generation sentinel reused from grain_target_cache: when the bank +# signal is unknown we treat every open as potentially stale and +# rebuild eagerly. Mirrors the semantics used elsewhere. +_GEN_UNKNOWN = salt.utils.grain_target_cache._GEN_UNKNOWN # noqa: SLF001 + + +def _encode_path_segments(segments): + """ + Return the ``\\x1e``-joined byte encoding of ``segments``. + + Segments are coerced to ``str`` then UTF-8 encoded. Empty-string + segments are preserved so colon-in-value predicates (like + ``ipv6:fe80::1``) can still be partitioned faithfully by the + query-time splitter. + """ + encoded = [] + for seg in segments: + if isinstance(seg, bytes): + encoded.append(seg) + else: + encoded.append(str(seg).encode("utf-8", errors="replace")) + return _PATH_SEP.join(encoded) + + +def _encode_scalar_value(value): + """ + Return the byte encoding used for a scalar leaf value. + + ``str`` passes through as UTF-8. ``bool``, ``int``, ``float`` use + Python's default ``str()`` so we round-trip the human-readable form + that a grain-target predicate also uses. + """ + if isinstance(value, bytes): + return value + if isinstance(value, bool): + # ``bool`` is a subclass of ``int`` in Python; check first so + # ``True`` -> ``"True"`` matches what a grain predicate carries. + return str(value).encode("utf-8") + return str(value).encode("utf-8", errors="replace") + + +def _build_key(path_bytes, value_bytes): + """Return the raw mmap key for a ``(path, value)`` pair.""" + return path_bytes + _PATH_VALUE_SEP + value_bytes + + +def _split_predicate(expr, delimiter): + """ + Enumerate every valid ``(path_segments, value_string)`` partition of + ``expr`` against ``delimiter``. + + For ``expr = "a:b:c"``, ``delimiter = ":"`` this yields:: + + (["a"], "b:c") + (["a", "b"], "c") + + Note: empty trailing segments are NOT emitted because a predicate + ``"a:b:"`` with no value cannot match any leaf. Empty MIDDLE + segments ARE preserved so that ``ipv6:fe80::1`` is partitioned as + ``(["ipv6"], "fe80::1")``, ``(["ipv6", "fe80"], ":1")``, + ``(["ipv6", "fe80", ""], "1")`` -- the exact set of lookups + described in the mission edge-case table. + """ + if not isinstance(expr, str) or not expr: + return + segments = expr.split(delimiter) + if len(segments) < 2: + return + for i in range(1, len(segments)): + path = segments[:i] + value = delimiter.join(segments[i:]) + if not value: + # Last segment partition with empty value is not useful -- + # a trie leaf is always a non-empty string value. + continue + yield path, value + + +def _flatten_leaves(gdict): + """ + Yield ``(path_bytes, value_bytes)`` pairs for every indexable + scalar leaf under ``gdict``. + + Walk rules (mirrors the mission spec): + + * ``dict`` -> recurse, path extended with the key. + * ``list`` / + ``tuple`` -> recurse into each element at the SAME path + so e.g. ``nameservers = ["1.1.1.1", "8.8.8.8"]`` + produces two leaves both under + ``("dns", "nameservers")``. + * ``None`` -> skip. + * ``str`` / ``int`` / + ``float`` / + ``bool`` -> yield one leaf. + * ``bytes`` -> yield one leaf (passed through verbatim). + * anything else -> skip with a DEBUG log. + + Does NOT emit for the top-level call when ``gdict`` is not a dict. + """ + if not isinstance(gdict, dict): + return + stack = [((), gdict)] + while stack: + path, node = stack.pop() + if isinstance(node, dict): + for k, v in node.items(): + stack.append((path + (k,), v)) + elif isinstance(node, (list, tuple)): + for element in node: + stack.append((path, element)) + elif node is None: + continue + elif isinstance(node, (str, int, float, bool, bytes)): + try: + path_bytes = _encode_path_segments(path) + value_bytes = _encode_scalar_value(node) + except Exception: # pylint: disable=broad-except + log.debug( + "mmap_trie_index: skipping unencodable leaf path=%r value=%r", + path, + node, + exc_info=True, + ) + continue + yield path_bytes, value_bytes + else: + log.debug( + "mmap_trie_index: skipping unsupported leaf type %s at path=%r", + type(node).__name__, + path, + ) + + +def _pack_list(sr_keys): + """Serialise an sr_key list to msgpack bytes.""" + return msgpack.packb(list(sr_keys), use_bin_type=True) + + +def _pack_cap(count): + """ + Serialise the legacy cap sentinel to msgpack bytes. + + Retained only so :mod:`tests` and the back-compat decode path can + construct an old-format sentinel; no production call site writes + these any more (Mission 6 chunks over-cap buckets instead). + """ + return msgpack.packb({"_capped": True, "count": int(count)}, use_bin_type=True) + + +def _pack_chunk_marker(n_chunks, total): + """ + Serialise a chunk-marker entry to msgpack bytes. + + ``n_chunks`` is the number of chunk entries stored under derived + keys, ``total`` is the sr_key count across every chunk. + """ + return msgpack.packb( + {"_chunks": int(n_chunks), "_total": int(total)}, use_bin_type=True + ) + + +def _chunk_write_fresh(key_bytes, sr_keys, max_list): + """ + Return the ``(items, n_chunks, total, promoted)`` tuple describing + the write-set for a bucket whose current on-disk state is empty / + a legacy sentinel. + + ``items`` is a list of ``(key, packed_value)`` pairs ready to feed + into :meth:`MmapCache.put_many`. ``promoted`` is ``True`` when the + resulting layout is chunked (i.e. the input exceeded ``max_list``). + + The function never reads the backing cache; it is purely a layout + decision. + """ + total = len(sr_keys) + if total <= max_list: + return ( + [(bytes(key_bytes), _pack_list(sr_keys))], + 0, + total, + False, + ) + items = [] + n_chunks = 0 + for start in range(0, total, max_list): + chunk = sr_keys[start : start + max_list] + items.append((_chunk_key(key_bytes, n_chunks), _pack_list(chunk))) + n_chunks += 1 + items.append((bytes(key_bytes), _pack_chunk_marker(n_chunks, total))) + return items, n_chunks, total, True + + +def _chunk_key(key_bytes, chunk_index): + """ + Return the storage key for a specific chunk of a wide bucket. + + The key is the natural bucket key with ``\\x1d`` + an 8-character + uppercase hex rendering of the chunk index appended. Hex encoding + (as opposed to a binary ``to_bytes``) avoids embedding ``\\x00`` + into the key -- :class:`MmapCache` uses NUL as the key-field + padding sentinel and truncates reads at the first null, which + would silently collapse ``chunk_index=0`` and all subsequent + chunks onto the same slot. 8 hex chars encode up to ``2**32`` + chunks (far beyond any realistic fleet's ``2**32 * max_list_size`` + sr_keys per bucket). + """ + if not isinstance(key_bytes, (bytes, bytearray)): + raise TypeError("key_bytes must be bytes") + idx = int(chunk_index) + if idx < 0 or idx > 0xFFFFFFFF: + raise ValueError(f"chunk_index out of range: {idx!r}") + return bytes(key_bytes) + _CHUNK_SEP + f"{idx:08X}".encode("ascii") + + +def _unpack_entry(raw): + """ + Decode a trie entry value. + + Returns one of: + + * ``("list", [sr_key, ...])`` for a regular hit. + * ``("chunks", {"_chunks": N, "_total": T})`` for a chunk marker; + the caller must then fetch each chunk and union them. + * ``("cap", {"_capped": True, "count": N})`` for the legacy cap + sentinel (on-disk back-compat only; no new writes produce this). + * ``None`` when the bytes cannot be decoded or carry an unknown + shape. + """ + if raw is None: + return None + if isinstance(raw, str): + raw = raw.encode("utf-8", errors="replace") + if not isinstance(raw, (bytes, bytearray)): + return None + if not raw: + return ("list", []) + try: + obj = msgpack.unpackb(bytes(raw), raw=False) + except Exception: # pylint: disable=broad-except + log.debug("mmap_trie_index: undecodable entry (%d bytes)", len(raw)) + return None + if isinstance(obj, dict): + if "_chunks" in obj: + return ("chunks", obj) + if obj.get("_capped"): + return ("cap", obj) + # Unknown dict shape -- corrupt entry. + log.warning( + "mmap_trie_index: unknown dict-shaped entry keys=%r; treating as missing", + sorted(obj.keys()), + ) + return None + if isinstance(obj, list): + return ("list", obj) + # Unknown shape -- treat as missing so the caller falls back. + return None + + +class MmapTrieIndex: + """ + Persistent trie-style scalar-leaf index backed by + :class:`salt.utils.mmap_cache.MmapCache`. + + One instance per ``(cachedir, bank)`` pair; callers normally + acquire instances through :func:`for_bank`, which maintains a + module-level registry so opening the same bank twice returns the + same object (and the same ``MmapCache`` fds). + + The class only *reads* the primary bank; it never mutates it. It + writes its own three mmap-cache files under ``/.trie/``. + """ + + def __init__( + self, + cachedir, + bank, + max_list_size=DEFAULT_MAX_LIST_SIZE, + batch_size=DEFAULT_BATCH_SIZE, + mmap_size=_DEFAULT_MMAP_SIZE, + slot_size=_DEFAULT_SLOT_SIZE, + key_size=_DEFAULT_KEY_SIZE, + max_segment_bytes=None, + ): + self.cachedir = cachedir + self.bank = bank + self.max_list_size = max_list_size + self.batch_size = batch_size + self.mmap_size = mmap_size + self.slot_size = slot_size + self.key_size = key_size + # ``None`` means "inherit MmapCache's default" (1 GiB). Tests + # exercising the segment-roll code path override this to force + # quick rolls on tiny data. + self.max_segment_bytes = max_segment_bytes + + self._trie_dir = os.path.join(cachedir, os.path.normpath(bank), _TRIE_DIRNAME) + self._index_path = os.path.join(self._trie_dir, _TRIE_INDEX_FILENAME) + self._meta_path = os.path.join(self._trie_dir, _TRIE_META_FILENAME) + # MmapCache reused across queries; opened lazily and reopened + # after a rebuild swaps the underlying files out (the inode + # changes, MmapCache's staleness check will tear down the stale + # mmap on next ``open()``). + self._mmap_cache = None + self._meta_cache = None # dict from msgpack.unpackb of meta file + self._lock = threading.RLock() + # Guards concurrent rebuild attempts from racing on the + # ``.new`` temp files. + self._build_lock = threading.Lock() + + # ------------------------------------------------------------------ + # File-level helpers + # ------------------------------------------------------------------ + + def _ensure_dir(self): + """Create the ``/.trie/`` directory if missing.""" + os.makedirs(self._trie_dir, exist_ok=True) + + def _mmap_cache_kwargs(self): + """ + Return extra kwargs to forward to :class:`MmapCache` from + either the live reader (:meth:`_get_mmap`) or the rebuild's + temp cache. Optional tunables that the operator didn't set are + omitted so the ``MmapCache`` defaults apply. + """ + kwargs = {} + if self.max_segment_bytes is not None: + kwargs["max_segment_bytes"] = self.max_segment_bytes + return kwargs + + def _get_mmap(self): + """ + Return an open :class:`MmapCache` for the trie index, creating + and opening it on first use. + + Returns ``None`` if the index file does not exist yet AND we + cannot create one (e.g. read-only filesystem). Callers treat + ``None`` as "no trie available, fall through". + """ + if self._mmap_cache is not None: + return self._mmap_cache + self._ensure_dir() + try: + cache = salt.utils.mmap_cache.MmapCache( + path=self._index_path, + size=self.mmap_size, + slot_size=self.slot_size, + key_size=self.key_size, + **self._mmap_cache_kwargs(), + ) + except Exception: # pylint: disable=broad-except + log.warning( + "mmap_trie_index: cannot construct MmapCache at %s", + self._index_path, + exc_info=True, + ) + return None + self._mmap_cache = cache + return cache + + def _reset_mmap(self): + """Close and forget the current MmapCache (post-rebuild swap).""" + if self._mmap_cache is not None: + try: + self._mmap_cache.close() + except Exception: # pylint: disable=broad-except + pass + self._mmap_cache = None + + def _list_new_heap_segments(self, base_new_heap_path, include_base=False): + """ + Return the rolled-over scratch heap segments on disk for the + in-progress rebuild. + + The result is sorted by segment ID ascending. When + ``include_base`` is True the segment-0 file (``base_new_heap_path`` + itself) is included at the head of the list if it exists, so the + caller can iterate the full segment set in one pass. + + Result shape: + + * ``include_base=False`` -> ``[str, str, ...]`` -- just the paths + of ``.new.heap.1``, ``.new.heap.2``, ... that exist. + * ``include_base=True`` -> ``[(seg_id, path), ...]`` sorted by + ``seg_id`` ascending. + """ + directory = os.path.dirname(base_new_heap_path) or "." + base_name = os.path.basename(base_new_heap_path) + prefix = base_name + "." + found = [] + try: + entries = os.listdir(directory) + except OSError: + entries = [] + for name in entries: + if not name.startswith(prefix): + continue + suffix = name[len(prefix) :] + if not suffix.isdigit(): + continue + seg_id = int(suffix) + found.append((seg_id, os.path.join(directory, name))) + found.sort(key=lambda item: item[0]) + if include_base: + if os.path.exists(base_new_heap_path): + return [(0, base_new_heap_path)] + found + return found + return [p for _seg_id, p in found] + + def _list_live_heap_segment_ids(self, base_heap_path): + """ + Return the segment IDs (sorted ascending) currently backing the + live trie heap: 0 for the base file, plus any rolled-over + ``.heap.N`` siblings. Missing files are omitted (an aborted + prior rebuild may leave gaps). + """ + ids = [] + if os.path.exists(base_heap_path): + ids.append(0) + directory = os.path.dirname(base_heap_path) or "." + base_name = os.path.basename(base_heap_path) + prefix = base_name + "." + try: + entries = os.listdir(directory) + except OSError: + entries = [] + for name in entries: + if not name.startswith(prefix): + continue + suffix = name[len(prefix) :] + if not suffix.isdigit(): + continue + ids.append(int(suffix)) + ids.sort() + return ids + + def _read_meta(self): + """ + Return the persisted meta dict for the current trie, or ``None`` + when the meta file is missing / unreadable / corrupt. + """ + try: + with salt.utils.files.fopen(self._meta_path, "rb") as fh: + raw = fh.read() + except (FileNotFoundError, PermissionError, OSError): + return None + if not raw: + return None + try: + obj = msgpack.unpackb(raw, raw=False) + except Exception: # pylint: disable=broad-except + log.warning( + "mmap_trie_index: meta file %s is corrupt -- forcing rebuild", + self._meta_path, + ) + return None + if not isinstance(obj, dict): + return None + return obj + + def _write_meta_atomic(self, meta): + """ + Write ``meta`` (a dict) to the meta file via a temp file + + ``os.replace`` so observers never see a half-written header. + """ + self._ensure_dir() + raw = msgpack.packb(meta, use_bin_type=True) + fd, tmp = tempfile.mkstemp(dir=self._trie_dir, prefix=".trie_meta_") + try: + with os.fdopen(fd, "wb") as fh: + fh.write(raw) + fh.flush() + os.fsync(fh.fileno()) + fd = -1 + os.replace(tmp, self._meta_path) + finally: + if fd != -1: + try: + os.close(fd) + except OSError: + pass + try: + if os.path.exists(tmp): + os.remove(tmp) + except OSError: + pass + + def _generation_tag(self, current_gen): + """ + Return a JSON-friendly serialisation of a ``_bank_generation`` + tuple so it survives msgpack round-trips in the meta file. + + ``_bank_generation`` returns one of ``("mmap", int)``, + ``("dir", int)``, or the ``_GEN_UNKNOWN`` sentinel. + """ + if current_gen is _GEN_UNKNOWN: + return None + if isinstance(current_gen, tuple) and len(current_gen) == 2: + return [current_gen[0], current_gen[1]] + return None + + def _meta_matches(self, meta, current_gen): + """Return True if ``meta`` recorded the same generation we see now.""" + if meta is None: + return False + if current_gen is _GEN_UNKNOWN: + return False + stored = meta.get("generation") + wanted = self._generation_tag(current_gen) + if stored is None or wanted is None: + return False + return list(stored) == wanted + + @property + def generation(self): + """ + Return a monotonic generation tag for the current trie. + + Returns the ``built_at_ns`` nanosecond timestamp from the meta + file: strictly monotonic across successive rebuilds and always + an ``int`` when the trie has been built. Callers use this to + detect "did the trie change since I last looked?" with a cheap + ``>=`` comparison. + + Returns ``None`` only when the trie has never been built (no + meta file on disk) or the meta file is unreadable / corrupt. + + O(1): reads a cached dict already loaded by + :meth:`ensure_current`; falls back to a single meta-file read + when accessed before ``ensure_current``. + """ + meta = self._meta_cache + if meta is None: + meta = self._read_meta() + if meta is not None: + self._meta_cache = meta + if not meta: + return None + built_at_ns = meta.get("built_at_ns") + if built_at_ns is None: + return None + return int(built_at_ns) + + # ------------------------------------------------------------------ + # Build pipeline + # ------------------------------------------------------------------ + + def ensure_current(self, cache, bank): + """ + Rebuild the on-disk trie if the bank's generation has advanced. + + Called at the top of :meth:`lookup_literal` and + :meth:`values_for_path`. Returns ``True`` if a current trie is + available, ``False`` if we could not produce one (e.g. the + primary bank has no mmap or the rebuild failed). + """ + try: + current_gen = ( + salt.utils.grain_target_cache._bank_generation( # noqa: SLF001 + cache, bank + ) + ) + except Exception: # pylint: disable=broad-except + current_gen = _GEN_UNKNOWN + + if self._meta_cache is None: + self._meta_cache = self._read_meta() + + if self._meta_matches(self._meta_cache, current_gen): + return self._get_mmap() is not None + + # Serialise concurrent build attempts; the second arrival will + # find the meta current when it re-checks. + with self._build_lock: + self._meta_cache = self._read_meta() + if self._meta_matches(self._meta_cache, current_gen): + return self._get_mmap() is not None + try: + self._rebuild(cache, bank, current_gen) + except Exception: # pylint: disable=broad-except + log.warning( + "mmap_trie_index: rebuild of bank=%r failed; " + "retaining stale/absent trie", + bank, + exc_info=True, + ) + return self._get_mmap() is not None + return self._get_mmap() is not None + + def _iter_bank_rows(self, cache, bank): + """ + Yield ``(sr_key, grain_dict)`` pairs from *cache* / *bank* in a + streaming fashion. + + Prefers per-key fetch over ``list_all``: ``list_all`` builds the + full mapping in memory (defeats the memory-discipline goal). + If the driver does not expose ``list`` we return an empty + iterator and log at DEBUG. + """ + try: + keys = cache.list(bank) or [] + except Exception: # pylint: disable=broad-except + log.debug( + "mmap_trie_index: cache.list(%r) failed; " + "no trie entries will be built", + bank, + exc_info=True, + ) + return + for sr_key in keys: + try: + gdict = cache.fetch(bank, sr_key) + except Exception: # pylint: disable=broad-except + continue + yield sr_key, gdict + + def _rebuild(self, cache, bank, current_gen): + """ + Produce a fresh trie by streaming the primary bank through the + flattener, batching into a bounded accumulator, and atomically + swapping the built files into place. + + Memory ceiling: one grain dict + the accumulator (bounded by + ``self.batch_size``). The accumulator is a dict mapping encoded + keys to lists of sr_keys; its size is bounded by ``batch_size`` + TOTAL sr_keys across all buckets. + + Exceptions mid-build leave the temp files behind; the finally + clause cleans them up. The live trie is only mutated once the + whole pass succeeds. + """ + self._ensure_dir() + new_index_path = self._index_path + ".new" + # MmapCache-adjacent files follow the ``.heap`` / ``.roster`` + # naming scheme the class uses internally. + new_heap_path = new_index_path + ".heap" + new_roster_path = new_index_path + ".roster" + # Remove any stale .new files from a previously-aborted build. + # This includes rolled-over heap segments ``.new.heap.1``, + # ``.new.heap.2``, ... -- MmapCache rolls to a new segment + # after each 1 GiB of heap data by default. + stale_new_files = [new_index_path, new_heap_path, new_roster_path] + stale_new_files.extend(self._list_new_heap_segments(new_heap_path)) + for p in stale_new_files: + if os.path.exists(p): + try: + os.remove(p) + except OSError: + pass + + try: + new_cache = salt.utils.mmap_cache.MmapCache( + path=new_index_path, + size=self.mmap_size, + slot_size=self.slot_size, + key_size=self.key_size, + **self._mmap_cache_kwargs(), + ) + except Exception: # pylint: disable=broad-except + log.warning( + "mmap_trie_index: cannot create temp MmapCache at %s; aborting rebuild", + new_index_path, + exc_info=True, + ) + return + + # Force creation of the backing files even if the bank is + # empty. Without this, an empty-bank rebuild would skip the + # atomic swap and leave stale .idx / .heap / .roster in place + # from a previous non-empty generation. + if not new_cache.open(write=True): + log.warning( + "mmap_trie_index: cannot initialise temp MmapCache files at %s", + new_index_path, + ) + try: + new_cache.close() + except Exception: # pylint: disable=broad-except + pass + return + + accumulator = {} + pending_count = 0 + # ``chunked_count`` is a cheap counter exposed in the log line; + # no per-key dict is kept in process memory. Promotion to + # chunked layout is decided at flush time by comparing + # ``len(existing_bucket) + len(new_sr_keys)`` against + # ``max_list_size`` -- the mmap entry is the source of truth, + # so the builder needs no O(unique-keys) observed-counts dict. + chunked_count = 0 + total_entries = 0 + total_rows = 0 + t0 = time.monotonic() + + try: + for sr_key, gdict in self._iter_bank_rows(cache, bank): + total_rows += 1 + if not isinstance(gdict, dict): + continue + sr_key_str = str(sr_key) + for path_bytes, value_bytes in _flatten_leaves(gdict): + key = _build_key(path_bytes, value_bytes) + bucket = accumulator.get(key) + if bucket is None: + bucket = [] + accumulator[key] = bucket + bucket.append(sr_key_str) + pending_count += 1 + if pending_count >= self.batch_size: + chunked_count += self._flush(new_cache, accumulator) + total_entries += len(accumulator) + accumulator.clear() + pending_count = 0 + if accumulator: + chunked_count += self._flush(new_cache, accumulator) + total_entries += len(accumulator) + accumulator.clear() + except Exception: # pylint: disable=broad-except + log.warning( + "mmap_trie_index: rebuild loop failed mid-stream for bank=%r", + bank, + exc_info=True, + ) + # Abort: close and delete temp files; leave live state alone. + try: + new_cache.close() + except Exception: # pylint: disable=broad-except + pass + abort_cleanup = [new_index_path, new_roster_path, new_heap_path] + abort_cleanup.extend(self._list_new_heap_segments(new_heap_path)) + for p in abort_cleanup: + try: + if os.path.exists(p): + os.remove(p) + except OSError: + pass + return + + try: + new_cache.close() + except Exception: # pylint: disable=broad-except + pass + + # Atomic swap: heap segments (all of them!) -> roster -> index. + # ``os.replace`` is atomic on POSIX; the index file is swapped + # last so an observer that races in during the swap sees the + # pre-rebuild index + its matching pre-rebuild heap, never a + # new index pointing into a deleted heap. The generation tag + # in the meta file is updated last of all, so partial swaps + # don't look "current" to the next reader -- they trigger a + # full rebuild. + # + # Heap segments: ``MmapCache`` rolls heap data to additional + # files once segment 0 exceeds ``max_segment_bytes`` (1 GiB by + # default). The swap MUST cover every rolled segment: an + # operator with >1 GiB of trie-heap data would otherwise find + # the index's segment-N offsets pointing at a freshly renamed + # segment 0 that doesn't contain them. + live_heap_path = self._index_path + ".heap" + new_segments = self._list_new_heap_segments(new_heap_path, include_base=True) + # Validate the segment numbering: segments must be contiguous + # starting at 0 (seg 0 == new_heap_path, seg N == .new.heap.N). + # A gap indicates a torn build; refuse to swap. + expected_ids = list(range(len(new_segments))) + if [seg_id for seg_id, _ in new_segments] != expected_ids: + observed = [seg_id for seg_id, _ in new_segments] + log.error( + "mmap_trie_index: rebuild for bank=%r produced non-contiguous " + "heap segments %r; refusing swap to avoid a torn trie", + bank, + observed, + ) + for _seg_id, p in new_segments: + try: + if os.path.exists(p): + os.remove(p) + except OSError: + pass + for p in (new_roster_path, new_index_path): + try: + if os.path.exists(p): + os.remove(p) + except OSError: + pass + return + + # Discover the full set of live segments too, so we can prune + # any that the new build no longer uses (e.g. cardinality cap + # shrank the data to fit in segment 0). + old_segment_ids = self._list_live_heap_segment_ids(live_heap_path) + + swap_pairs = [] + for seg_id, src in new_segments: + if seg_id == 0: + dst = live_heap_path + else: + dst = f"{live_heap_path}.{seg_id}" + swap_pairs.append((src, dst)) + if os.path.exists(new_roster_path): + swap_pairs.append((new_roster_path, self._index_path + ".roster")) + if os.path.exists(new_index_path): + swap_pairs.append((new_index_path, self._index_path)) + + # Close any live reader we hold so the inode swap is visible. + self._reset_mmap() + try: + for src, dst in swap_pairs: + os.replace(src, dst) + except OSError as exc: + log.warning( + "mmap_trie_index: failed atomic swap for bank=%r (%s); " + "trie may be inconsistent until next successful rebuild", + bank, + exc, + ) + # Clean up any left-over temp files. + for src, _dst in swap_pairs: + try: + if os.path.exists(src): + os.remove(src) + except OSError: + pass + return + + # Prune stale live heap segments whose offset range is no + # longer referenced by the swapped-in index. This handles the + # rare case where a cardinality cap shrank the heap enough that + # the new build emitted fewer segments than the previous one; + # leaving them in place would waste disk and -- worse -- let a + # future ``MmapCache._discover_segments`` reattach them if the + # inode numbering lined up suspiciously. + max_new_seg_id = new_segments[-1][0] if new_segments else -1 + for seg_id in old_segment_ids: + if seg_id <= max_new_seg_id: + continue + stale = f"{live_heap_path}.{seg_id}" + try: + if os.path.exists(stale): + os.remove(stale) + except OSError: + log.debug( + "mmap_trie_index: could not remove stale segment %s", + stale, + exc_info=True, + ) + + meta = { + "generation": self._generation_tag(current_gen), + "built_at_ns": time.time_ns(), + "max_list_size": int(self.max_list_size), + "batch_size": int(self.batch_size), + "entries": int(total_entries), + "rows": int(total_rows), + } + try: + self._write_meta_atomic(meta) + self._meta_cache = meta + except OSError as exc: + log.warning( + "mmap_trie_index: failed to write meta for bank=%r: %s", + bank, + exc, + ) + self._meta_cache = None + + log.info( + "mmap_trie_index: rebuilt trie for bank=%r in %.3fs " + "(rows=%d, entries=%d, chunked=%d)", + bank, + time.monotonic() - t0, + total_rows, + total_entries, + chunked_count, + ) + + def _flush(self, dest_cache, accumulator): + """ + Merge every key in *accumulator* into *dest_cache* via + ``put_many`` (one flock + batched fsync). + + **Merge semantics**: the streaming builder flushes periodically + to stay inside its memory budget, so the same ``(path, value)`` + key may already have a partial list stored from an earlier + flush within the same rebuild. Per-key behaviour: + + * No existing entry: write a plain msgpack list if the new + bucket fits in ``max_list_size``; otherwise write it as a + chunked bucket (``N`` chunk entries + one chunk-marker on + the natural key). + * Existing ``list``: extend in-memory, then write back as a + plain list -- unless the merged size exceeds + ``max_list_size``, in which case the bucket is *promoted* to + chunked storage (all existing + new entries redistributed + into chunks and the marker written over the natural key). + * Existing ``chunks``: we only touch the LAST chunk. If it has + free space, append until full; roll over to a new chunk once + full; repeat until all new entries land. Earlier chunks are + immutable across the rebuild, which is what makes cumulative + merge cost O(M) linear instead of O(M^2 / batch_size). + * Existing ``cap`` (legacy sentinel): treated as "needs + rebuild", replaced in-place by the new bucket. We land the + new entries as if the slot were empty; a follow-up flush in + the same rebuild will see the new list / chunks naturally. + + Returns the count of buckets that newly became chunked during + this flush (purely for logging; not required for correctness). + """ + if not accumulator: + return 0 + items = [] + newly_chunked = 0 + max_list = self.max_list_size + for key, bucket in accumulator.items(): + existing_raw = None + try: + existing_raw = dest_cache.get(key) + except Exception: # pylint: disable=broad-except + existing_raw = None + kind = "none" + obj = None + if existing_raw is not None and existing_raw is not True: + decoded = _unpack_entry(existing_raw) + if decoded is not None: + kind, obj = decoded + else: + kind = "none" + + new_list = list(bucket) + + if kind == "chunks": + # Append to an existing chunked bucket by mutating only + # the last chunk. Collect per-key writes; the final + # ``put_many`` below batches them with everything else. + n_chunks = int(obj.get("_chunks", 0) or 0) + total = int(obj.get("_total", 0) or 0) + if n_chunks <= 0: + # Marker claimed no chunks -- treat as fresh. + chunk_items, n_chunks, total, promoted = _chunk_write_fresh( + key, new_list, max_list + ) + items.extend(chunk_items) + if promoted: + newly_chunked += 1 + continue + # Read the last chunk (the only one we might mutate). + last_idx = n_chunks - 1 + last_chunk_key = _chunk_key(key, last_idx) + last_chunk = [] + try: + raw_chunk = dest_cache.get(last_chunk_key) + except Exception: # pylint: disable=broad-except + raw_chunk = None + if raw_chunk is not None and raw_chunk is not True: + decoded_chunk = _unpack_entry(raw_chunk) + if decoded_chunk is not None and decoded_chunk[0] == "list": + last_chunk = list(decoded_chunk[1]) + # Drain the new entries into the last chunk, rolling + # over as it fills. + for sr_key in new_list: + if len(last_chunk) < max_list: + last_chunk.append(sr_key) + else: + # Seal the current last chunk, start a new one. + items.append((last_chunk_key, _pack_list(last_chunk))) + n_chunks += 1 + last_idx = n_chunks - 1 + last_chunk_key = _chunk_key(key, last_idx) + last_chunk = [sr_key] + total += 1 + # Always write the final last chunk and an updated marker. + items.append((last_chunk_key, _pack_list(last_chunk))) + items.append((key, _pack_chunk_marker(n_chunks, total))) + continue + + if kind == "list": + merged = list(obj) + new_list + if len(merged) > max_list: + chunk_items, _n, _t, _promoted = _chunk_write_fresh( + key, merged, max_list + ) + items.extend(chunk_items) + newly_chunked += 1 + else: + items.append((key, _pack_list(merged))) + continue + + # kind in ("none", "cap") -- treat the slot as empty. For + # the legacy ``cap`` case this discards the sentinel and + # replaces it with freshly-written data (the stored sr_keys + # were already lost; we rebuild from scratch). + chunk_items, _n, _t, promoted = _chunk_write_fresh(key, new_list, max_list) + items.extend(chunk_items) + if promoted: + newly_chunked += 1 + + if not items: + return newly_chunked + try: + dest_cache.put_many(items) + except Exception: # pylint: disable=broad-except + log.warning( + "mmap_trie_index: put_many failed during flush " + "(%d items); entries may be missing until next rebuild", + len(items), + exc_info=True, + ) + return newly_chunked + + # ------------------------------------------------------------------ + # Query API + # ------------------------------------------------------------------ + + def _get_entry(self, key_bytes): + """ + Fetch a raw trie entry and decode it. + + Returns the decoded ``(kind, obj)`` tuple (see + :func:`_unpack_entry`), or ``None`` if the key is absent or + unreadable. + """ + mm = self._get_mmap() + if mm is None: + return None + try: + raw = mm.get(key_bytes) + except Exception: # pylint: disable=broad-except + log.debug("mmap_trie_index: get(%r) failed", key_bytes, exc_info=True) + return None + if raw is None or raw is True: + # ``raw is True`` means a presence-only entry; we never + # write those, so treat as missing. + return None + return _unpack_entry(raw) + + def lookup_literal(self, path_segments, value): + """ + Return the sr_key list for an exact ``(path, value)`` match. + + Return value semantics mirror the mission spec: + + * ``list`` (possibly empty) -- the trie served the predicate. + * ``None`` -- the entry is a legacy cap sentinel (stored data + was lost) or references orphaned chunks, and the caller must + fall back to the slow path for this partition. + + A missing key returns an empty list (``[]``) -- the trie knows + the key does not exist, so no fall-back is required. + + For a chunked bucket the sr_key lists from each chunk are + concatenated in insertion order (chunk 0 first). Duplicate + suppression is NOT performed here: within a single rebuild the + builder guarantees no cross-chunk duplicates, and callers of + :func:`dispatch_predicate` already deduplicate across partitions. + """ + path_bytes = _encode_path_segments(path_segments) + value_bytes = _encode_scalar_value(value) + key_bytes = _build_key(path_bytes, value_bytes) + entry = self._get_entry(key_bytes) + if entry is None: + return [] + kind, obj = entry + if kind == "cap": + # Legacy sentinel left over from an older build: data was + # thrown away. Signal fall-back; next rebuild will chunk it. + log.debug( + "mmap_trie_index: legacy cap sentinel at key=%r; " + "triggering slow-path fallback until next rebuild", + key_bytes, + ) + return None + if kind == "chunks": + return self._read_chunks(key_bytes, obj) + return list(obj) + + def _read_chunks(self, key_bytes, marker): + """ + Materialise a chunked bucket by reading every chunk entry and + concatenating their sr_key lists. + + Returns ``None`` when a chunk entry is missing or carries an + unexpected shape (an orphaned marker / corrupt chunk). Logs a + WARNING so operators can catch chronic issues; the caller then + falls through to the slow path. + """ + mm = self._get_mmap() + if mm is None: + return None + try: + n_chunks = int(marker.get("_chunks", 0) or 0) + except (TypeError, ValueError): + n_chunks = 0 + if n_chunks <= 0: + # Marker with zero chunks -- treat as an empty bucket. + return [] + out = [] + for i in range(n_chunks): + chunk_key = _chunk_key(key_bytes, i) + try: + raw = mm.get(chunk_key) + except Exception: # pylint: disable=broad-except + log.warning( + "mmap_trie_index: get of chunk %d for key=%r raised; " + "treating bucket as unavailable", + i, + key_bytes, + exc_info=True, + ) + return None + if raw is None or raw is True: + log.warning( + "mmap_trie_index: orphaned chunk marker at key=%r " + "(chunk %d/%d missing); falling back to slow path", + key_bytes, + i, + n_chunks, + ) + return None + decoded = _unpack_entry(raw) + if decoded is None or decoded[0] != "list": + log.warning( + "mmap_trie_index: chunk %d of key=%r has unexpected " + "shape (kind=%r); falling back to slow path", + i, + key_bytes, + None if decoded is None else decoded[0], + ) + return None + out.extend(decoded[1]) + return out + + def values_for_path(self, path_segments): + """ + Return every scalar value observed under *path_segments* across + all SRs. + + Walks the mmap roster and filters by the exact path prefix + (``\\x00``). Yields decoded ``value_bytes`` as UTF-8 + strings. Non-UTF-8 values pass through as the raw bytes; the + caller is expected to accept either. + + This is the glob / regex support path -- the caller walks the + returned set and applies ``fnmatch`` / ``re.search`` per value, + then calls :meth:`lookup_literal` per match. + """ + mm = self._get_mmap() + if mm is None: + return [] + path_bytes = _encode_path_segments(path_segments) + wanted_prefix = path_bytes + _PATH_VALUE_SEP + prefix_len = len(wanted_prefix) + out = [] + seen = set() + try: + keys = mm.list_keys() or [] + except Exception: # pylint: disable=broad-except + log.debug( + "mmap_trie_index: list_keys failed in values_for_path", exc_info=True + ) + return [] + for k in keys: + if isinstance(k, str): + kb = k.encode("utf-8", errors="replace") + else: + kb = bytes(k) + if not kb.startswith(wanted_prefix): + continue + value_bytes = kb[prefix_len:] + # Skip chunk sub-keys: a chunked bucket stores entries + # under ``\x1d``. The natural key + # (which is also present on disk, carrying the chunk + # marker) will still be emitted by the plain prefix match + # below, so there is no information loss from skipping the + # chunk-key sub-entries here. + if _CHUNK_SEP in value_bytes: + continue + try: + value_str = value_bytes.decode("utf-8") + except UnicodeDecodeError: + value_str = value_bytes + if value_str in seen: + continue + seen.add(value_str) + out.append(value_str) + return out + + def close(self): + """Release resources. Safe to call repeatedly.""" + self._reset_mmap() + + +# -------------------------------------------------------------------- +# Module-level registry +# -------------------------------------------------------------------- + +_registry_lock = threading.RLock() +_registry: dict = {} + + +def for_bank(cache, bank, opts): + """ + Return the :class:`MmapTrieIndex` for ``(cache.cachedir, bank)``, + creating it on first use. + + ``opts`` supplies the tuning knobs: + + * ``grain_trie_max_list_size`` -- default 100000 + * ``grain_trie_build_batch_size`` -- default 1024 + * ``grain_trie_mmap_size`` -- MmapCache slot count, default 2000000 + * ``grain_trie_mmap_max_segment_bytes`` -- per-segment heap cap + forwarded to :class:`MmapCache`; ``None`` / absent inherits the + 1 GiB default. + + When the cache driver does not expose ``cachedir`` (some test + fakes) the function returns ``None`` -- the caller falls back to + the slow path. + """ + cachedir = getattr(cache, "cachedir", None) + if not cachedir: + return None + max_list = int(opts.get("grain_trie_max_list_size", DEFAULT_MAX_LIST_SIZE) or 0) + if max_list <= 0: + max_list = DEFAULT_MAX_LIST_SIZE + batch = int(opts.get("grain_trie_build_batch_size", DEFAULT_BATCH_SIZE) or 0) + if batch <= 0: + batch = DEFAULT_BATCH_SIZE + # ``grain_trie_mmap_size`` sets the MmapCache slot count. Each slot + # is ``slot_size`` bytes (default 160) so the sparse index file is + # ``mmap_size * slot_size`` bytes. Tests shrink this to avoid + # allocating a 320 MiB sparse file per tmp bank -- a fleet of them + # on tmpfs-backed CI runners fills ``/tmp`` and silently drops + # writes. + mmap_size_raw = opts.get("grain_trie_mmap_size") + try: + mmap_size = int(mmap_size_raw) if mmap_size_raw else None + except (TypeError, ValueError): + mmap_size = None + if mmap_size is not None and mmap_size <= 0: + mmap_size = None + # ``grain_trie_mmap_max_segment_bytes`` lets the operator tune (or, + # in tests, shrink) the per-segment heap cap passed to + # :class:`MmapCache`. ``None`` / absent means "inherit the mmap + # default" (1 GiB). + max_seg_raw = opts.get("grain_trie_mmap_max_segment_bytes") + try: + max_seg = int(max_seg_raw) if max_seg_raw else None + except (TypeError, ValueError): + max_seg = None + if max_seg is not None and max_seg <= 0: + max_seg = None + reg_key = (cachedir, bank) + with _registry_lock: + entry = _registry.get(reg_key) + if entry is not None: + # If the operator tuned the knobs, honour the new values + # on an existing instance (max_list_size + batch_size only + # affect future rebuilds). + entry.max_list_size = max_list + entry.batch_size = batch + entry.max_segment_bytes = max_seg + return entry + kwargs = { + "cachedir": cachedir, + "bank": bank, + "max_list_size": max_list, + "batch_size": batch, + "max_segment_bytes": max_seg, + } + if mmap_size is not None: + kwargs["mmap_size"] = mmap_size + idx = MmapTrieIndex(**kwargs) + _registry[reg_key] = idx + return idx + + +def invalidate(cachedir=None, bank=None): + """ + Drop registry entries. With no arguments drops every tracked bank. + """ + with _registry_lock: + if cachedir is None and bank is None: + for idx in list(_registry.values()): + with contextlib.suppress(Exception): + idx.close() + _registry.clear() + return + reg_key = (cachedir or "", bank) + entry = _registry.pop(reg_key, None) + if entry is not None: + with contextlib.suppress(Exception): + entry.close() + + +# -------------------------------------------------------------------- +# Dispatch helper used by salt.utils.minions +# -------------------------------------------------------------------- + + +def dispatch_predicate(cache, bank, opts, expr, delimiter, regex_match): + """ + Attempt to answer a grain-target predicate from the mmap trie. + + Returns one of: + + * ``list`` (possibly empty) -- UNION of sr_keys that match every + valid ``(path, value)`` partition of ``expr``. Caller should + consume this and short-circuit without the slow path. Wide + buckets (``> max_list_size``) are transparently reconstituted by + reading chunk entries off disk, so a single trie partition can + now return millions of sr_keys without triggering fall-back. + * ``None`` -- the trie could not serve the predicate: e.g. the + index is unavailable, the opts switch is off, a partition hit a + *legacy* cap sentinel (data was lost in an older build and the + trie is out-of-date), the chunk marker was orphaned, or the + predicate shape didn't yield any valid partition. Caller must + fall through. + + Legacy cap sentinel back-compat: a Mission-4-era on-disk trie may + still contain ``{"_capped": True, ...}`` entries. Hitting one + returns ``None`` -- the next successful rebuild will re-materialise + the bucket as chunked and the slow-path fallback will not fire + again for that key. + """ + if not opts.get("enable_grain_trie_index"): + return None + if not isinstance(expr, str) or delimiter not in expr: + return None + idx = for_bank(cache, bank, opts) + if idx is None: + return None + if not idx.ensure_current(cache, bank): + return None + + out = [] + seen = set() + any_partition_served = False + + for path_segments, value_str in _split_predicate(expr, delimiter): + if regex_match: + served, hits = _serve_regex_partition(idx, path_segments, value_str) + elif _is_literal_pattern(value_str): + served, hits = _serve_literal_partition(idx, path_segments, value_str) + else: + served, hits = _serve_glob_partition(idx, path_segments, value_str) + if not served: + # Cap or lookup failure -- fall back entirely. + log.debug( + "mmap_trie_index: partition path=%r value=%r could not be " + "served (cap hit); falling back to slow path", + path_segments, + value_str, + ) + return None + any_partition_served = True + for sr_key in hits: + if sr_key not in seen: + out.append(sr_key) + seen.add(sr_key) + + # No valid partition at all -> predicate wasn't shaped like + # ``:``; caller must use the slow path. + if not any_partition_served: + return None + return out + + +def _is_literal_pattern(value): + """``True`` iff value has no fnmatch wildcards.""" + if not isinstance(value, str) or not value: + return False + return not any(c in value for c in ("*", "?", "[")) + + +def _serve_literal_partition(idx, path_segments, value_str): + """ + Return ``(served, hits)`` for a literal-value partition. + + ``served = False`` means the trie could not materialise the + bucket (legacy cap sentinel or orphaned chunk marker) and the + caller must bail. ``served = True`` with an empty ``hits`` is a + definitive "no match" and the caller can continue to the next + partition. + """ + hits = idx.lookup_literal(path_segments, value_str) + if hits is None: + return False, [] + return True, hits + + +def _serve_glob_partition(idx, path_segments, pattern): + """Return ``(served, hits)`` for a glob (fnmatch) value partition.""" + try: + values = idx.values_for_path(path_segments) + except Exception: # pylint: disable=broad-except + log.debug( + "mmap_trie_index: values_for_path failed for glob partition", + exc_info=True, + ) + return False, [] + hits = [] + seen = set() + for value in values: + if not isinstance(value, str): + continue + if not fnmatch.fnmatchcase(value, pattern): + continue + sr_keys = idx.lookup_literal(path_segments, value) + if sr_keys is None: + # Individual value's bucket is unavailable (legacy cap + # sentinel or orphaned chunks) -- bail the whole partition. + # A correct answer would need to see those excluded sr_keys + # too. + return False, [] + for sr_key in sr_keys: + if sr_key not in seen: + hits.append(sr_key) + seen.add(sr_key) + return True, hits + + +def _serve_regex_partition(idx, path_segments, pattern): + """Return ``(served, hits)`` for a PCRE value partition.""" + try: + rx = re.compile(pattern) + except re.error: + # Unparseable regex -- caller's slow path will hit the same + # error, but returning (False, []) keeps our behaviour + # consistent with "could not serve, fall back". + return False, [] + try: + values = idx.values_for_path(path_segments) + except Exception: # pylint: disable=broad-except + log.debug( + "mmap_trie_index: values_for_path failed for regex partition", + exc_info=True, + ) + return False, [] + hits = [] + seen = set() + for value in values: + if not isinstance(value, str): + continue + if not rx.search(value): + continue + sr_keys = idx.lookup_literal(path_segments, value) + if sr_keys is None: + return False, [] + for sr_key in sr_keys: + if sr_key not in seen: + hits.append(sr_key) + seen.add(sr_key) + return True, hits diff --git a/salt/utils/resource_registry.py b/salt/utils/resource_registry.py index 4a2f69dcb57a..29ae102a648f 100644 --- a/salt/utils/resource_registry.py +++ b/salt/utils/resource_registry.py @@ -96,6 +96,15 @@ RESOURCE_BANK = "resources" +#: Logical data_type name used by the cluster routing table to ring-shard +#: resource-registry writes. Not a ``salt.cache.Cache`` bank -- the +#: resource registry bypasses that facade -- this name is only the key +#: operators pass to ``salt-run cluster.ring_create`` / +#: ``cluster.route_set`` to turn on 1-of-N routing for the register fan-out. +#: Consulted by :func:`salt.cache._cluster_replicator.fire_resource_register_write` +#: via :func:`salt.cluster.ring_membership.get_ring`. +RESOURCE_INDEX_RING_NAME = "resource_index" + #: Cache bank holding per-resource grain dicts indexed by composite SRN #: ``":"``. Populated by the master's ``_register_resources`` #: handler from the load shipped by @@ -138,6 +147,18 @@ #: rate is a first-order performance concern on hot write paths. DEFAULT_COMPACT_MIN_INTERVAL = 30.0 +#: Default TTL (seconds) for an entry in the chunked-register pending window +#: before it is evicted as stale. Protects against a minion that ships +#: chunks ``0..k`` and then crashes before ``chunk_final=True``: without +#: eviction the per-minion pending set accumulates indefinitely, pinning +#: SRNs in process memory. The TTL window starts at first-chunk arrival +#: and refreshes on every subsequent chunk for the same ``(minion_id, +#: chunk_scope)`` pair, so an active-but-slow register never evicts its +#: own window. Configurable via ``cluster_resource_register_pending_ttl`` +#: in master opts. Default of 5 minutes is comfortably larger than the +#: observed 200K-register window (~5 min on a clustered deployment). +DEFAULT_PENDING_REGISTERS_TTL = 300.0 + #: Persisted schema version for the in-cache fallback dict (legacy path). RESOURCE_INDEX_SCHEMA_VERSION = 2 @@ -692,6 +713,72 @@ def __init__(self, opts, store=None, cache=None): ) self._last_compact_check = 0.0 + # ------------------------------------------------------------------ + # Chunked register accumulator (used by chunked register RPC) + # ------------------------------------------------------------------ + # + # The minion-side register loop ships resource inventory in + # per-type chunks (see + # :meth:`salt.minion.Minion._register_resources_with_master`). + # Each non-final chunk must additively ``put`` its SRNs with NO + # delete pass -- a mid-stream delete-diff against a previous full + # inventory would prune already-persisted earlier chunks of the + # same registration window. Deletes only run on the terminal + # (``chunk_final=True``) chunk, computed against the union of + # every SRN seen during the window for that ``(minion_id, rtype)`` + # pair. + # + # ``self._pending_registers`` is per-minion state keyed by + # ``minion_id`` holding the cumulative SRN set for the current + # in-progress registration window. The entry is cleared when the + # final chunk commits. Each chunk is tagged with a scope -- + # either ``"__all__"`` (legacy single-shot register that replaces + # every type at once) or ``""`` (chunked per-type register) + # -- so the terminal delete only prunes within the types the + # minion is actually re-registering. A minion that re-registers + # only ``dummy`` must not wipe its previously-registered ``ssh`` + # entries. + # + # Per-scope layout (new): the window state for each scope is a + # ``dict`` with keys: + # + # * ``keys`` -- cumulative set of SRNs seen in this window. + # * ``seen_indexes`` -- set of ``chunk_index`` values observed. + # * ``final_pending`` -- ``None`` if no final chunk has arrived + # yet, else the ``chunk_index`` of the final chunk (the maximum + # index of the window). A non-``None`` value means an + # out-of-order ``chunk_final=True`` landed before the full + # ``0..final_pending`` sequence was in; the prune is deferred + # until the missing indices fill in. + # * ``created_ts`` -- ``time.monotonic()`` when the first chunk + # for this ``(minion, scope)`` landed. Used by the TTL + # sweep to detect crash-loop-abandoned windows. + # * ``last_activity_ts`` -- ``time.monotonic()`` when the most + # recent chunk for this scope arrived. A long gap between + # now and this timestamp triggers eviction via + # :meth:`_evict_stale_pending_registers_locked`. Refreshing + # on activity means active-but-slow registers never evict + # their own window. + # + # ``_pending_registers_lock`` guards both the map structure and + # the per-scope mutations (``put_many``, prune) so concurrent + # peer-side apply threads cannot interleave a prune pass against + # a partially-populated window. Peer-side arrival ordering is + # now FIFO per minion (see + # :func:`salt.cache._cluster_replicator._get_peer_apply_executor`), + # so this lock is primarily cross-process (originator MWorker + # vs. peer apply) and TTL-sweep safety. + self._pending_registers = {} + self._pending_registers_lock = threading.Lock() + # Per-entry TTL for stale pending-window eviction. See + # :data:`DEFAULT_PENDING_REGISTERS_TTL` for rationale. + self._pending_registers_ttl = float( + opts.get( + "cluster_resource_register_pending_ttl", + DEFAULT_PENDING_REGISTERS_TTL, + ) + ) + def close(self): """ Close the backing mmap so temp dirs can be removed and FDs are not @@ -880,6 +967,231 @@ def register_minion(self, minion_id, resources): return len(to_put), len(to_delete) + def register_minion_chunk( + self, minion_id, resources, chunk_final, chunk_scope=None, chunk_index=None + ): + """ + Chunk-aware register: additively ``put`` the entries in ``resources`` + and only prune stale SRNs on the terminal chunk of a registration + window. + + Multiple calls with ``chunk_final=False`` accumulate SRNs in + ``self._pending_registers[minion_id]`` without computing any + delete-diff. A single call with ``chunk_final=True`` commits the + remaining SRNs and then runs the prune pass against the union of + every SRN seen in the window. + + ``chunk_scope`` selects which of the previously-registered SRNs + the terminal chunk is allowed to delete: + + * ``None`` or ``"__all__"`` -- legacy single-shot semantics; the + final chunk prunes against ALL previous SRNs for this minion + (equivalent to :meth:`register_minion`). + * ```` -- scoped per-type register; the final chunk prunes + only previously-registered SRNs of ``rtype``. This preserves + the invariant that re-registering one type's inventory does + not wipe the minion's resources of other types. + + ``chunk_index`` is used to detect and buffer out-of-order chunk + arrivals on peer masters (where the publish-daemon dispatches + each ``cluster/peer/delegate-resource-register-write`` event + into the default thread pool via ``run_in_executor`` -- the + executor gives no FIFO guarantee). If a ``chunk_final=True`` + arrives with gaps in the ``0..chunk_index`` sequence, the prune + is deferred until the missing indices land. Legacy callers + that omit ``chunk_index`` get pre-chunk semantics (prune on + every ``chunk_final=True``, no gap check). + + :param str minion_id: The reporting minion. + :param dict resources: ``{resource_type: [resource_id, ...]}`` -- + the slice carried by this chunk. May be empty for a + type-less terminal chunk that only exists to trigger prune. + :param bool chunk_final: True iff this chunk ends a registration + window. Only the terminal chunk runs the delete pass. + :param str chunk_scope: see above. + :param int chunk_index: Monotonically-increasing index assigned + by the minion-side sender (0-based within the window). + ``None`` disables the gap check (used by legacy single-shot + callers). + :returns: ``(n_put, n_deleted)`` for logging. ``n_deleted`` is + always zero on non-final chunks AND on terminal chunks whose + prune was deferred because of a missing earlier index. + """ + scope = chunk_scope or "__all__" + # Materialize this chunk's SRN set and ``put_many`` payload. + new_keys = set() + to_put = [] + for rtype, rids in (resources or {}).items(): + for rid in rids or (): + srn = resource_index_srn_key(rtype, rid) + new_keys.add(srn) + to_put.append((srn, minion_id, rtype)) + # The mmap write is itself fsync'd and independent per SRN, so + # it can safely run outside the lock (duplicate ``put_many`` is + # idempotent). But accumulating into ``_pending_registers`` and + # computing the prune MUST be under the lock -- a concurrent + # peer-apply thread for the same ``(minion_id, scope)`` could + # otherwise observe a half-populated ``keys`` set and wipe + # valid SRNs. + if to_put: + self._store.put_many(to_put) + + now = time.monotonic() + with self._pending_registers_lock: + # Evict stale pending windows (crash-loop minions that + # shipped some non-final chunks then disconnected). Runs + # under the write-path lock so no background task is + # required; amortized cost is O(number of live pending + # scopes), which is small (bounded by N_minions * N_types). + self._evict_stale_pending_registers_locked(now) + + pending = self._pending_registers.setdefault(minion_id, {}) + scope_state = pending.get(scope) + if scope_state is None: + scope_state = { + "keys": set(), + "seen_indexes": set(), + "final_pending": None, + "created_ts": now, + "last_activity_ts": now, + } + pending[scope] = scope_state + else: + scope_state["last_activity_ts"] = now + scope_state["keys"].update(new_keys) + if chunk_index is not None: + scope_state["seen_indexes"].add(chunk_index) + + # If a final chunk has already arrived out-of-order and this + # chunk fills a gap, re-check readiness so the deferred + # prune can fire now. Otherwise honor the caller's + # ``chunk_final`` flag. + final_index = scope_state["final_pending"] + if chunk_final and chunk_index is not None: + # New terminal arrival: record its index, pick the max + # if a prior terminal already landed (defensive -- the + # minion-side sender only ships one terminal per window). + final_index = ( + chunk_index + if final_index is None + else max(final_index, chunk_index) + ) + scope_state["final_pending"] = final_index + elif chunk_final and chunk_index is None: + # Legacy single-shot path -- no index-based gap check. + # Treat as "ready to prune" below. + scope_state["final_pending"] = -1 + final_index = -1 + + # Determine whether the terminal chunk is ready to prune. + # Ready iff a final has arrived (``final_pending`` not + # ``None``) AND either the legacy sentinel (``-1``) applies, + # OR every index 0..final_index has been observed. + ready_to_prune = False + if final_index is not None: + if final_index == -1: + ready_to_prune = True + else: + expected = set(range(final_index + 1)) + ready_to_prune = expected.issubset(scope_state["seen_indexes"]) + + if not ready_to_prune: + if chunk_final: + log.debug( + "register_minion_chunk: deferring terminal prune for " + "minion=%s scope=%s final_index=%s seen_indexes=%s " + "(out-of-order chunk arrival)", + minion_id, + scope, + final_index, + sorted(scope_state["seen_indexes"]), + ) + return len(to_put), 0 + + # Ready to prune: snapshot the committed SRN set and clear + # the per-scope entry (and the minion entry if it's now + # empty) before releasing the lock. The ``delete_many`` + # call below runs outside the lock so it doesn't hold up + # concurrent registrations for other minions. + committed = scope_state["keys"] + del pending[scope] + if not pending: + self._pending_registers.pop(minion_id, None) + + # Prune stale SRNs within the window's scope. + previous = self._store.resources_by_minion(minion_id) + to_delete = [] + for rtype, rids in previous.items(): + if scope != "__all__" and rtype != scope: + continue + for rid in rids: + srn = resource_index_srn_key(rtype, rid) + if srn not in committed: + to_delete.append(srn) + if to_delete: + self._store.delete_many(to_delete) + + # Opportunistic compaction on the write path -- same policy as + # :meth:`register_minion`. + self.maybe_compact() + + return len(to_put), len(to_delete) + + def _evict_stale_pending_registers_locked(self, now): + """ + Evict ``_pending_registers`` entries whose last activity is + older than :attr:`_pending_registers_ttl` seconds. + + Called from :meth:`register_minion_chunk` on the write path + while holding :attr:`_pending_registers_lock`. No background + task required: every new chunk arrival runs the sweep and + drops any abandoned windows that belong to a different + ``(minion_id, chunk_scope)`` pair. + + Each eviction is logged at WARNING level so operators notice + crash-loop minions. The eviction is intentionally quiet when + the pending map is empty (no log spam on cold start). + + :param float now: Current ``time.monotonic()`` reading. + Threaded in rather than re-read so the caller's single + clock sample is reused for both the sweep and the + incoming-chunk ``last_activity_ts`` update, avoiding + a boundary where the just-added entry could be evicted + on its own creation call. + """ + ttl = self._pending_registers_ttl + if ttl <= 0 or not self._pending_registers: + return + stale_entries = [] + for minion_id, scopes in self._pending_registers.items(): + for scope, state in scopes.items(): + last_activity = state.get("last_activity_ts") or state.get("created_ts") + if last_activity is None: + # Legacy-shape entry predating TTL tracking: stamp + # it now so the next sweep can age it correctly. + state["last_activity_ts"] = now + continue + age = now - last_activity + if age > ttl: + stale_entries.append((minion_id, scope, age, len(state["keys"]))) + for minion_id, scope, age, srn_count in stale_entries: + log.warning( + "resource_registry: evicting stale pending-register window " + "minion=%s scope=%s age=%.1fs srn_count=%d (TTL=%.1fs). " + "Minion likely crashed mid-register or never sent chunk_final.", + minion_id, + scope, + age, + srn_count, + ttl, + ) + scopes = self._pending_registers.get(minion_id) + if scopes is None: + continue + scopes.pop(scope, None) + if not scopes: + self._pending_registers.pop(minion_id, None) + def unregister_minion(self, minion_id): """ Tombstone every primary entry that maps to ``minion_id``. Used when diff --git a/salt/utils/resources.py b/salt/utils/resources.py index 15e1f3d0668e..8958302501d1 100644 --- a/salt/utils/resources.py +++ b/salt/utils/resources.py @@ -1,5 +1,27 @@ """ -Helpers for Salt resource minions: configurable pillar key and lookups. +Helpers for Salt resource minions. + +Resource inventory is declared on the minion via a top-level minion opt +``resource_types`` (list of type names) plus whatever per-type opts each +module needs (e.g. ``dummy_resource_count`` for the dummy module). Pillar +is **not** the source of resource inventory -- holding a 200K-resource +grain dict in ``opts["pillar"]["resources"]`` violates the minion memory +ceiling (see ``feedback-no-resource-data-in-pillar`` and +``feedback-minion-never-holds-full-resource-set``). + +Resource module ``discover(opts)`` contract:: + + discover(opts) -> Iterable[str] + + Yield bare resource IDs this minion manages for a given type. + MAY be a generator; the register loop consumes it exactly once. + Implementations SHOULD NOT buffer the full set in memory when + the source is sequential/countable. + + The caller (minion ``_register_resources_with_master``) is + responsible for chunking. A single call to ``discover()`` + returns one iterable for the full inventory; chunking happens + at the register-loop level. """ import logging @@ -7,44 +29,32 @@ log = logging.getLogger(__name__) -def resource_pillar_key(opts): +def enabled_resource_types(opts): """ - Return the top-level pillar key used for per-type resource configuration. + Return the list of resource types this minion is configured to load. - Configured by minion option ``resource_pillar_key`` (default ``resources``). - Empty values are rejected with a warning and treated as ``"resources"``. + Configured by the minion option ``resource_types`` -- a list of + strings. Empty / missing / wrong-shape values yield an empty list + (minion manages no resources). """ - key = opts.get("resource_pillar_key", "resources") - if not key: + types = opts.get("resource_types") or [] + if not isinstance(types, list): log.warning( - "resource_pillar_key is empty; using default 'resources'. " - "Set resource_pillar_key to a non-empty string in the minion config." + "resource_types must be a list; got %r -- treating as empty", + type(types).__name__, ) - key = "resources" - return key - - -def pillar_resources_tree(opts): - """ - Return the merged pillar mapping under the configured resource pillar key. - - If the key is absent, returns ``{}`` (same as an empty declaration). - Non-dict values are treated as empty. - """ - key = resource_pillar_key(opts) - pillar = opts.get("pillar", {}) - if key not in pillar: - return {} - pr = pillar.get(key) - return pr if isinstance(pr, dict) else {} + return [] + return [t for t in types if isinstance(t, str) and t] def bare_resource_ids_from_decl(decl): """ - Flatten ``salt_resources`` / pillar ``resources`` subtree mappings to bare IDs. + Flatten ``salt_resources`` grains-subtree mappings to bare IDs. Supports ``{rtype: [id, ...]}`` (post-discovery) and - ``{rtype: {"resource_ids": [...]}}`` (pillar-shaped) layouts. + ``{rtype: {"resource_ids": [...]}}`` (legacy-shaped) layouts. + Used by the grains-cache fallback in + :func:`bare_resource_id_in_minion_data_cache`. """ if not isinstance(decl, dict): return [] @@ -61,50 +71,29 @@ def bare_resource_ids_from_decl(decl): def bare_resource_id_in_minion_data_cache(opts, resource_id, cache=None): """ - Return ``True`` if ``resource_id`` appears in any cached minion pillar / + Return ``True`` if ``resource_id`` appears in any cached minion grains snapshot on the master. - Used when the mmap resource registry has not yet recorded the ID (or was - cleared) but :conf_master:`minion_data_cache` still holds pillar/grains - from the last sync — so ``salt m2-dummy2 state.apply`` / ``test.ping`` - style targeting can still resolve. + Used when the mmap resource registry has not yet recorded the ID (or + was cleared) but :conf_master:`minion_data_cache` still holds grains + from the last sync -- so ``salt m2-dummy2 state.apply`` / + ``test.ping`` style targeting can still resolve for offline minions. + + The pillar-reading branch of this helper was removed when pillar + stopped being the source of resource inventory; the grains-cache + branch (``salt_resources`` grain) remains as the only fallback. :param cache: - Optional :class:`salt.cache.Cache` from the caller (e.g. ``CkMinions`` - already constructs one). Pillar entries may live under a separate - backend when :conf_master:`pillar.cache_driver` is set; that driver is - always used for ``bank="pillar"`` reads. Grains use ``cache`` (or a - newly created default cache handle when ``cache`` is omitted). + Optional :class:`salt.cache.Cache` from the caller (e.g. + :class:`salt.utils.minions.CkMinions` already constructs one). + A new default cache is created when ``cache`` is omitted. """ if not opts.get("minion_data_cache") or not resource_id: return False try: - import salt.cache + import salt.cache # noqa: PLC0415 grains_cache = cache if cache is not None else salt.cache.factory(opts) - pillar_driver = opts.get("pillar.cache_driver") - pillar_cache = ( - salt.cache.factory(opts, driver=pillar_driver) - if pillar_driver - else grains_cache - ) - rk = resource_pillar_key(opts) - try: - p_keys = pillar_cache.list("pillar") - except Exception: # pylint: disable=broad-except - p_keys = [] - for mid in p_keys: - try: - data = pillar_cache.fetch("pillar", mid) or {} - except Exception: # pylint: disable=broad-except - continue - if not isinstance(data, dict): - continue - subtree = data.get(rk) - ids = bare_resource_ids_from_decl(subtree) - if resource_id in ids: - return True - try: g_keys = grains_cache.list("grains") except Exception: # pylint: disable=broad-except diff --git a/tests/monitoring/cluster-3node/Dockerfile.salt b/tests/monitoring/cluster-3node/Dockerfile.salt new file mode 100644 index 000000000000..d1f66e2a03de --- /dev/null +++ b/tests/monitoring/cluster-3node/Dockerfile.salt @@ -0,0 +1,64 @@ +# Cluster-5node variant of ../Dockerfile.salt — drops the `pip install memray` +# step (debug/profiling only, not needed for cluster tests, and the default +# docker-build network has been failing to reach pypi for that step on this +# host). Keep everything else identical so this image is a superset-compatible +# drop-in for cluster scenario testing. + +FROM python:3.10-slim + +RUN apt-get update && apt-get install -y \ + build-essential \ + libssl-dev \ + libffi-dev \ + python3-dev \ + procps \ + curl \ + wget \ + libzmq3-dev \ + tini \ + gdb \ + iproute2 \ + zlib1g-dev \ + libbz2-dev \ + liblzma-dev \ + libsqlite3-dev \ + libreadline-dev \ + libncurses-dev \ + && rm -rf /var/lib/apt/lists/* + +ARG PYTHON_VERSION=3.10.20 +RUN cd /tmp && \ + wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz && \ + tar xf Python-${PYTHON_VERSION}.tar.xz && \ + cd Python-${PYTHON_VERSION} && \ + ./configure \ + --enable-shared \ + --prefix=/usr/local \ + --with-ensurepip=install \ + CFLAGS="-g -O2 -fno-omit-frame-pointer" && \ + make -j"$(nproc)" && \ + make install && \ + ldconfig && \ + cd / && rm -rf /tmp/Python-${PYTHON_VERSION}* + +WORKDIR /app + +COPY requirements/ /app/requirements/ +COPY setup.py /app/ +COPY pyproject.toml /app/ +COPY MANIFEST.in /app/ +COPY README.rst /app/ +COPY AUTHORS /app/ +COPY LICENSE /app/ +COPY NOTICE /app/ +COPY salt/ /app/salt/ +COPY tools/ /app/tools/ +COPY scripts/ /app/scripts/ + +RUN pip install --no-cache-dir -r requirements/base.txt -r requirements/zeromq.txt +RUN pip install --no-cache-dir -e . + +RUN useradd -m -s /bin/bash salt && echo "salt:salt" | chpasswd +RUN usermod -aG shadow salt + +ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/salt-master"] diff --git a/tests/monitoring/cluster-3node/README.md b/tests/monitoring/cluster-3node/README.md new file mode 100644 index 000000000000..79c7d5ddac14 --- /dev/null +++ b/tests/monitoring/cluster-3node/README.md @@ -0,0 +1,49 @@ +# cluster-3node testbed + +3-master Salt cluster + haproxy LB + 1 scale minion (200K dummy +resources) for Phase-3 memory-ceiling + ring-mode verification. + +## Topology + +- `c3-salt-master-1 / 2 / 3`: 3-master Raft cluster in + `cluster_id=scale_test`, `cluster_isolated_filesystem=True`. +- `c3-salt-haproxy`: TCP LB in front of the 3 masters. +- `c3-ring-setup`: one-shot sequencing shim. Creates the + `resource_index` ring + route + ring_set, waits for `active`, + exits 0. Runs BEFORE the minion so the minion's first + auto-register goes through the ring, not the broadcast fallback. +- `c3-salt-minion-1`: configured with + `resource_types: [dummy]` + `dummy_resource_count: 200000`. + +## Usage + +```bash +cd tests/monitoring/cluster-3node + +# From a worktree, point SALT_SRC at the worktree root so the +# container bind-mounts the branch's salt/ package. +export SALT_SRC=/home/you/src/salt # or /home/you/src/salt/.claude/worktrees/ +docker compose up -d + +# `up -d` returns only after c3-ring-setup exits 0 and the minion is up. +# First-register measurement: +docker exec c3-salt-minion-1 salt-call saltutil.refresh_resources wait=True timeout=600 +``` + +## Why the ring-setup shim + +Pre-shim, the minion's auto-register fired as soon as haproxy was up, +before any operator could manually create the `resource_index` ring. +That caused chunks 0..2 (30K SRNs out of 200K) to land via the +replicate fallback (full set to all 3 masters), polluting every master +with "shared" SRNs that the later ring-dispatched chunks could not +prune (the scope-terminal prune only removes SRNs NOT seen this window, +and the replicate-dupes had been seen on every master this window). + +The shim runs ring_create / route_set / ring_set before the minion +even starts, making `sum=200000 exact` the stable ring-mode invariant +for a fresh `up -d`. + +If you need to measure replicate mode, bring the stack up without the +ring (set `RING_NAME=` to disable the shim's create pass, or just +remove the ring state from the cluster_pki volume after `up -d`). diff --git a/tests/monitoring/cluster-3node/docker-compose.yml b/tests/monitoring/cluster-3node/docker-compose.yml new file mode 100644 index 000000000000..ca999a6f7e3e --- /dev/null +++ b/tests/monitoring/cluster-3node/docker-compose.yml @@ -0,0 +1,176 @@ +# 3-master cluster + haproxy LB + 1 scale minion monitoring stack. +# +# Ring-mode test-harness sequencing: +# +# The minion container used to start immediately after haproxy was up, +# which fired an auto-register BEFORE the operator had a chance to +# create the ``resource_index`` ring. That contaminated all 3 masters +# with 30K+ replicate-fallback SRNs that the later ring-dispatched +# register could not prune (they'd been written this window, so the +# terminal scope=dummy prune saw them as "live"). +# +# The ``c3-ring-setup`` shim below runs after all 3 masters are up, +# invokes the three ring runners against master-1, and exits. The +# minion waits on ``service_completed_successfully`` so its first +# auto-register goes through the ring, not the broadcast fallback. +# This makes ``sum=200000 exact`` the stable ring-mode invariant +# out of the box. +# +# Path parameterization: +# +# The ``SALT_SRC`` env var points at the salt source directory whose +# ``salt/`` package should be bind-mounted into the containers. The +# default resolves to the repo containing this compose file (via +# ``../../..``) so a plain ``docker compose up -d`` works from this +# directory. Verifiers running against a worktree override +# ``SALT_SRC`` to that worktree's root. + +x-master-common: &master-common + build: + context: ${SALT_SRC:-../../..} + dockerfile: tests/monitoring/cluster-3node/Dockerfile.salt + image: cluster-3node-salt:latest + entrypoint: ["/usr/bin/tini", "--"] + volumes: + - ${SALT_SRC:-../../..}/salt:/app/salt + - ./srv/salt:/srv/salt + - cluster-pki:/etc/salt/cluster-pki + - /etc/localtime:/etc/localtime:ro + networks: + - salt-net + +services: + salt-master-1: + <<: *master-common + container_name: c3-salt-master-1 + hostname: salt-master-1 + networks: + salt-net: + ipv4_address: 10.77.1.11 + volumes: + - ${SALT_SRC:-../../..}/salt:/app/salt + - ./srv/salt:/srv/salt + - ./master-1.conf:/etc/salt/master + - cluster-pki:/etc/salt/cluster-pki + - master-1-cache:/var/cache/salt/master + - /etc/localtime:/etc/localtime:ro + command: ["/usr/local/bin/salt-master", "-l", "debug"] + + salt-master-2: + <<: *master-common + container_name: c3-salt-master-2 + hostname: salt-master-2 + depends_on: + - salt-master-1 + networks: + salt-net: + ipv4_address: 10.77.1.12 + volumes: + - ${SALT_SRC:-../../..}/salt:/app/salt + - ./srv/salt:/srv/salt + - ./master-2.conf:/etc/salt/master + - cluster-pki:/etc/salt/cluster-pki + - master-2-cache:/var/cache/salt/master + - /etc/localtime:/etc/localtime:ro + command: ["/usr/local/bin/salt-master", "-l", "debug"] + + salt-master-3: + <<: *master-common + container_name: c3-salt-master-3 + hostname: salt-master-3 + depends_on: + - salt-master-1 + networks: + salt-net: + ipv4_address: 10.77.1.13 + volumes: + - ${SALT_SRC:-../../..}/salt:/app/salt + - ./srv/salt:/srv/salt + - ./master-3.conf:/etc/salt/master + - cluster-pki:/etc/salt/cluster-pki + - master-3-cache:/var/cache/salt/master + - /etc/localtime:/etc/localtime:ro + command: ["/usr/local/bin/salt-master", "-l", "debug"] + + salt-haproxy: + image: haproxy:2.8 + container_name: c3-salt-haproxy + hostname: salt-haproxy + depends_on: + - salt-master-1 + - salt-master-2 + - salt-master-3 + networks: + salt-net: + ipv4_address: 10.77.1.14 + volumes: + - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + + # Ring sequencing shim. Runs AFTER all 3 masters + haproxy are up. + # Invokes cluster.ring_create / route_set / ring_set against master-1 + # (which fans out to the Raft leader via cluster/peer/multi-ring-request + # so it works regardless of which master is leader), waits for the + # ring to become active, then exits 0. The minion ``depends_on: + # service_completed_successfully`` this service, so its first + # auto-register fires only after the ring is live. + # + # One-shot: restart policy is "no" and the container exits as soon as + # ``ring-setup.sh`` returns. ``docker compose up -d`` still returns + # after this exits successfully. + # The ring runners must publish events through master-1's live + # process (``/var/run/salt/master/master_event_pull.ipc``) and write + # job trackers under master-1's cache dir. A sidecar that only + # bind-mounts master-1.conf runs the runners in its own local state, + # so ``cluster.ring_create`` "succeeds" in the sidecar but never + # reaches the running master. Instead of trying to share master-1's + # cache + run dirs (race-prone), we exec the ring-setup script + # INSIDE master-1 via the host's docker socket. The sidecar only + # needs the ``docker`` CLI + the socket mount. + c3-ring-setup: + image: docker:27-cli + container_name: c3-ring-setup + hostname: c3-ring-setup + depends_on: + - salt-master-1 + - salt-master-2 + - salt-master-3 + - salt-haproxy + restart: "no" + networks: + - salt-net + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./ring-setup.sh:/usr/local/bin/ring-setup.sh:ro + entrypoint: ["/bin/sh"] + command: ["/usr/local/bin/ring-setup.sh"] + + salt-minion-1: + <<: *master-common + container_name: c3-salt-minion-1 + hostname: salt-minion-1 + depends_on: + c3-ring-setup: + condition: service_completed_successfully + networks: + salt-net: + ipv4_address: 10.77.1.21 + volumes: + - ${SALT_SRC:-../../..}/salt:/app/salt + - ./minion.conf:/etc/salt/minion + - minion-1-cache:/var/cache/salt/minion + - /etc/localtime:/etc/localtime:ro + command: ["/usr/local/bin/salt-minion", "-l", "debug"] + +networks: + salt-net: + driver: bridge + ipam: + config: + - subnet: 10.77.1.0/24 + +volumes: + cluster-pki: + master-1-cache: + master-2-cache: + master-3-cache: + minion-1-cache: diff --git a/tests/monitoring/cluster-3node/haproxy.cfg b/tests/monitoring/cluster-3node/haproxy.cfg new file mode 100644 index 000000000000..cfe52edc2cd1 --- /dev/null +++ b/tests/monitoring/cluster-3node/haproxy.cfg @@ -0,0 +1,33 @@ +global + maxconn 10000 + log stdout format raw local0 + +defaults + mode tcp + timeout connect 5s + timeout client 2h + timeout server 2h + option tcplog + log global + +# Minion ret_port (TCP req/rep) +frontend salt-ret + bind *:44506 + default_backend masters-ret + +backend masters-ret + balance roundrobin + server salt-master-1 salt-master-1:44506 check + server salt-master-2 salt-master-2:44506 check + server salt-master-3 salt-master-3:44506 check + +# Minion pub_port (TCP pub/sub) +frontend salt-pub + bind *:44505 + default_backend masters-pub + +backend masters-pub + balance roundrobin + server salt-master-1 salt-master-1:44505 check + server salt-master-2 salt-master-2:44505 check + server salt-master-3 salt-master-3:44505 check diff --git a/tests/monitoring/cluster-3node/master-1.conf b/tests/monitoring/cluster-3node/master-1.conf new file mode 100644 index 000000000000..bc7d8767a444 --- /dev/null +++ b/tests/monitoring/cluster-3node/master-1.conf @@ -0,0 +1,38 @@ +id: salt-master-1 +interface: 10.77.1.11 +publish_port: 44505 +ret_port: 44506 +open_mode: True +auto_accept: True +log_level: debug +log_granular_levels_disabled: + salt.master: debug + salt.cache._cluster_replicator: debug + salt.cluster: debug + salt.transport: info + +cluster_id: scale_test +cluster_isolated_filesystem: True +cluster_peers: + - salt-master-2 + - salt-master-3 +cluster_pki_dir: /etc/salt/cluster-pki + +file_roots: + base: + - /srv/salt/states + +pillar_roots: + base: + - /srv/salt/pillar/base + +worker_threads: 10 +ipc_write_buffer: 104857600 + +# Resource registry scale knobs +resource_registry_put_many_yield: 1024 + +# is "replicate" (N-of-N); set to "ring" to measure 1-of-N partitioning. + +# Lift payload cap for cluster/peer/delegate events (ring + replicate). +max_event_size: 268435456 diff --git a/tests/monitoring/cluster-3node/master-2.conf b/tests/monitoring/cluster-3node/master-2.conf new file mode 100644 index 000000000000..d18a4bf54492 --- /dev/null +++ b/tests/monitoring/cluster-3node/master-2.conf @@ -0,0 +1,36 @@ +id: salt-master-2 +interface: 10.77.1.12 +publish_port: 44505 +ret_port: 44506 +open_mode: True +auto_accept: True +log_level: debug +log_granular_levels_disabled: + salt.master: debug + salt.cache._cluster_replicator: debug + salt.cluster: debug + salt.transport: info + +cluster_id: scale_test +cluster_isolated_filesystem: True +cluster_peers: + - salt-master-1 + - salt-master-3 +cluster_pki_dir: /etc/salt/cluster-pki + +file_roots: + base: + - /srv/salt/states + +pillar_roots: + base: + - /srv/salt/pillar/base + +worker_threads: 10 +ipc_write_buffer: 104857600 + +resource_registry_put_many_yield: 1024 + + +# Lift payload cap for cluster/peer/delegate events (ring + replicate). +max_event_size: 268435456 diff --git a/tests/monitoring/cluster-3node/master-3.conf b/tests/monitoring/cluster-3node/master-3.conf new file mode 100644 index 000000000000..4fc0fa143ffe --- /dev/null +++ b/tests/monitoring/cluster-3node/master-3.conf @@ -0,0 +1,36 @@ +id: salt-master-3 +interface: 10.77.1.13 +publish_port: 44505 +ret_port: 44506 +open_mode: True +auto_accept: True +log_level: debug +log_granular_levels_disabled: + salt.master: debug + salt.cache._cluster_replicator: debug + salt.cluster: debug + salt.transport: info + +cluster_id: scale_test +cluster_isolated_filesystem: True +cluster_peers: + - salt-master-1 + - salt-master-2 +cluster_pki_dir: /etc/salt/cluster-pki + +file_roots: + base: + - /srv/salt/states + +pillar_roots: + base: + - /srv/salt/pillar/base + +worker_threads: 10 +ipc_write_buffer: 104857600 + +resource_registry_put_many_yield: 1024 + + +# Lift payload cap for cluster/peer/delegate events (ring + replicate). +max_event_size: 268435456 diff --git a/tests/monitoring/cluster-3node/minion.conf b/tests/monitoring/cluster-3node/minion.conf new file mode 100644 index 000000000000..afcae01eca85 --- /dev/null +++ b/tests/monitoring/cluster-3node/minion.conf @@ -0,0 +1,20 @@ +id: scale-minion-1 +master: salt-haproxy +master_port: 44506 +publish_port: 44505 +master_alive_interval: 30 +retry_dns: 10 +log_level: debug +ipc_write_buffer: 104857600 + +# Resource registry opts +resource_types: + - dummy +dummy_resource_count: 200000 +process_count_max: 5 +dummy_resource_persist_state: False +acceptance_wait_time: 2 +acceptance_wait_time_max: 10 +return_retry_timer: 30 +return_retry_timer_max: 60 +open_mode: True diff --git a/tests/monitoring/cluster-3node/ring-setup.sh b/tests/monitoring/cluster-3node/ring-setup.sh new file mode 100755 index 000000000000..238f72eeedaa --- /dev/null +++ b/tests/monitoring/cluster-3node/ring-setup.sh @@ -0,0 +1,147 @@ +#!/bin/sh +# Ring-mode sequencing shim for the cluster-3node testbed. +# +# Runs after all 3 masters + haproxy are up (compose ``depends_on`` +# ordering), creates the ``resource_index`` ring + route + ring_set +# INSIDE master-1 via ``docker exec``, polls until the ring is reported +# ``active``, then exits 0. +# +# Rationale for ``docker exec`` rather than a local ``salt-run``: +# the ring runners publish events through the live master's IPC +# socket (``/var/run/salt/master/master_event_pull.ipc``) and write +# per-jid trackers under the master's cache dir. A sidecar that +# only mounts ``master-1.conf`` runs the runners in its own local +# state, so ``cluster.ring_create`` "succeeds" in the sidecar but +# never reaches the running master-1 process. Executing inside +# master-1 is the only path that lets the runner publish to its +# own publisher daemon. +# +# The minion container uses ``depends_on: +# c3-ring-setup: service_completed_successfully`` so its first +# auto-register only fires AFTER this script exits cleanly. Without +# this gate the minion's startup auto-register races the operator's +# manual ``cluster.ring_create`` and the first 3 chunks (30K SRNs at +# 10K per chunk) land via replicate-fallback -- polluting every master +# with SRNs the subsequent ring-dispatched chunks can't prune. +# +# Any runner failure exits non-zero, which fails the compose wait so +# the minion refuses to come up -- better a fast red than a quiet +# ring-contamination that corrupts measurements. + +set -eu + +MASTER_CONTAINER="${SALT_MASTER_CONTAINER:-c3-salt-master-1}" +MASTER_HOST="${SALT_MASTER:-salt-master-1}" +RING_NAME="resource_index" +POLL_ATTEMPTS="${RING_READY_ATTEMPTS:-60}" +POLL_INTERVAL="${RING_READY_INTERVAL:-2}" + +log() { + echo "[ring-setup] $(date -u +%H:%M:%S) $*" +} + +m_exec() { + docker exec "$MASTER_CONTAINER" "$@" +} + +# Wait for master-1 to accept runner calls AND have RaftService wired up. +# The MWorker / publish-daemon forks take a few seconds after container +# start; RaftService start waits for the leader-election handshake on +# top of that. A ``cluster.ring_create`` event that arrives before +# RaftService is wired is dropped on the floor with a WARNING in the +# master log (``Multi-ring runner event ... arrived but RaftService +# is not wired up on this master; dropping``), which is silent at the +# shim's abstraction layer. We gate on ``cluster.members`` returning +# ``membership_version != -1`` (a committed CONFIG entry means the +# Raft log is accepting writes). +wait_for_master() { + attempt=0 + while [ "$attempt" -lt "$POLL_ATTEMPTS" ]; do + if m_exec salt-run --out=raw test.version >/dev/null 2>&1; then + log "master ${MASTER_CONTAINER} accepting runner calls" + break + fi + attempt=$((attempt + 1)) + sleep "$POLL_INTERVAL" + done + if [ "$attempt" -ge "$POLL_ATTEMPTS" ]; then + log "master never accepted runner calls after $((POLL_ATTEMPTS * POLL_INTERVAL))s" + return 1 + fi + + log "waiting for RaftService on ${MASTER_CONTAINER} (cluster.members membership_version != -1)" + attempt=0 + while [ "$attempt" -lt "$POLL_ATTEMPTS" ]; do + mv_line=$(m_exec salt-run --out=yaml cluster.members 2>/dev/null \ + | grep -E '^membership_version:' || true) + case "$mv_line" in + *': -1'|'') + ;; + *) + log "RaftService wired up (${mv_line})" + return 0 + ;; + esac + attempt=$((attempt + 1)) + sleep "$POLL_INTERVAL" + done + log "RaftService never came up after $((POLL_ATTEMPTS * POLL_INTERVAL))s" + log "final cluster.members output:" + m_exec salt-run cluster.members || true + return 1 +} + +create_ring() { + voter_list="['salt-master-1','salt-master-2','salt-master-3']" + log "cluster.ring_create name=${RING_NAME} voters=${voter_list}" + m_exec salt-run cluster.ring_create \ + "name=${RING_NAME}" \ + "voters=${voter_list}" || true + + log "cluster.route_set data_type=${RING_NAME} ring=${RING_NAME}" + m_exec salt-run cluster.route_set \ + "data_type=${RING_NAME}" \ + "ring=${RING_NAME}" || true + + log "cluster.ring_set name=${RING_NAME} members=voters" + m_exec salt-run cluster.ring_set \ + "name=${RING_NAME}" \ + "members=voters" || true +} + +# Poll ``cluster.rings`` until the ring appears in ``active_rings``. +# ``active_rings`` is the runner's own sorted list of ``status=active`` +# ring_ids -- a single check against that is simpler and less brittle +# than parsing the nested ``rings..status`` layout. +wait_ring_active() { + attempt=0 + while [ "$attempt" -lt "$POLL_ATTEMPTS" ]; do + if m_exec salt-run --out=yaml cluster.rings 2>/dev/null \ + | grep -E "^active_rings:" -A 5 \ + | grep -qE "^[[:space:]]*-[[:space:]]+${RING_NAME}[[:space:]]*\$"; then + log "ring '${RING_NAME}' is active" + return 0 + fi + attempt=$((attempt + 1)) + sleep "$POLL_INTERVAL" + done + log "ring never reached active state after $((POLL_ATTEMPTS * POLL_INTERVAL))s" + log "final cluster.rings output:" + m_exec salt-run cluster.rings || true + return 1 +} + +main() { + log "waiting for master ${MASTER_CONTAINER}..." + wait_for_master + + log "creating ring '${RING_NAME}'" + create_ring + + log "waiting for ring '${RING_NAME}' to become active" + wait_ring_active + + log "ring sequencing complete; exiting 0" +} + +main "$@" diff --git a/tests/monitoring/cluster-3node/srv/salt/pillar/base/top.sls b/tests/monitoring/cluster-3node/srv/salt/pillar/base/top.sls new file mode 100644 index 000000000000..42fd10b00189 --- /dev/null +++ b/tests/monitoring/cluster-3node/srv/salt/pillar/base/top.sls @@ -0,0 +1 @@ +base: {} diff --git a/tests/monitoring/cluster-3node/srv/salt/states/top.sls b/tests/monitoring/cluster-3node/srv/salt/states/top.sls new file mode 100644 index 000000000000..4e424867793b --- /dev/null +++ b/tests/monitoring/cluster-3node/srv/salt/states/top.sls @@ -0,0 +1,3 @@ +base: + '*': + [] diff --git a/tests/pytests/functional/cluster/consensus/test_cluster_ready_scenarios.py b/tests/pytests/functional/cluster/consensus/test_cluster_ready_scenarios.py index 8400bc6ae141..a9e9b03eff6b 100644 --- a/tests/pytests/functional/cluster/consensus/test_cluster_ready_scenarios.py +++ b/tests/pytests/functional/cluster/consensus/test_cluster_ready_scenarios.py @@ -31,6 +31,7 @@ import asyncio import multiprocessing +import tests.pytests.functional.cluster.consensus.test_raft_scenarios as _raft_scenarios_module from salt.cluster.consensus.service import RaftService from tests.pytests.functional.cluster.consensus.conftest import FakePusher from tests.pytests.functional.cluster.consensus.test_raft_scenarios import ( @@ -57,12 +58,26 @@ def _make_cluster_with_ready_tracking(node_ids, cachedir): Build a ``ServiceCluster`` where every service has an ``on_ready`` callback that appends the node-id to a shared ``fired`` list. + ``on_ready`` must be wired BEFORE ``RaftService.__init__`` returns + because ``__init__`` now drives ``apply_entries`` + ``reconcile_membership`` + on startup (see persisted ``commit_index`` fix in + ``salt/cluster/consensus/raft/log.py``). A node restarting against a + cachedir that already has the founding CONFIG committed fires + ``_on_ready`` during construction; a callback assigned to ``svc._on_ready`` + after construction would miss that fan-out. + + To wire the callback at construction time we monkey-patch the + ``RaftService`` symbol inside ``test_raft_scenarios`` (the module + whose ``ServiceCluster._init_service`` references it) for the + duration of ``ServiceCluster.__init__``, injecting an ``on_ready`` + keyword from a pre-built per-node map. + Returns ``(cluster, fired_list)``. """ fired = [] - cluster = ServiceCluster(node_ids, cachedir=cachedir) - for nid, svc in cluster.services.items(): + callbacks = {} + for nid in node_ids: nid_capture = nid def _make_callback(n): @@ -71,9 +86,21 @@ def _cb(): return _cb - svc._on_ready = _make_callback(nid_capture) - # Re-wire membership SM on_change so _on_membership_change runs - svc._node.membership_sm.on_change = svc._on_membership_change + callbacks[nid_capture] = _make_callback(nid_capture) + + original_raft_service = _raft_scenarios_module.RaftService + + class _ReadyTrackingRaftService(original_raft_service): + def __init__(self, opts, loop, peer_pushers, voting=True, on_ready=None): + if on_ready is None: + on_ready = callbacks.get(opts.get("id")) + super().__init__(opts, loop, peer_pushers, voting=voting, on_ready=on_ready) + + _raft_scenarios_module.RaftService = _ReadyTrackingRaftService + try: + cluster = ServiceCluster(node_ids, cachedir=cachedir) + finally: + _raft_scenarios_module.RaftService = original_raft_service for svc in cluster.services.values(): svc._node.become_follower() @@ -479,33 +506,44 @@ class TestClusterReadyOnRestart: """ Regression tests for the restart-with-persisted-state bootstrap bug. - Repro in the field: three cluster masters bootstrap cleanly on a - fresh cachedir, then are stopped and restarted with the same - ``cache/consensus`` / ``cache/raft`` state on disk. On restart the - Raft log is loaded from storage (founding CONFIG at index 0, term - N), but ``commit_index`` is not persisted so it comes back at -1 - and ``MembershipStateMachine`` is empty. A new leader is elected - in term N+1 but — per Raft §5.4.2 — cannot commit the index-0 - CONFIG by replication count alone because its term is less than - the leader's current term. Nothing new is proposed in steady - state so ``commit_index`` stays at -1 forever, ``_on_ready`` never - fires, and ``_cluster_is_ready`` returns ``False`` indefinitely. - - The fix: ``Node.become_leader`` appends a NOOP barrier in the new - leader's term when the log carries uncommitted prior-term entries. - Replicating + committing the NOOP transitively commits every prior - entry, including the restored founding CONFIG, which the - ``MembershipStateMachine`` then applies, firing ``on_change`` / - ``_on_ready`` on every node whose id is in the voter set. + Original repro in the field (now fixed): three cluster masters + bootstrap cleanly on a fresh cachedir, then are stopped and + restarted with the same ``cache/consensus`` / ``cache/raft`` state + on disk. Pre-fix, the Raft log was loaded from storage (founding + CONFIG at index 0, term N) but ``commit_index`` was not persisted + so it came back at -1 with an empty ``MembershipStateMachine``. + A new leader elected in term N+1 could not commit the index-0 + CONFIG by replication count alone (Raft §5.4.2 forbids committing + a prior-term entry that way), so ``commit_index`` stayed at -1 + forever, ``_on_ready`` never fired, and ``_cluster_is_ready`` + returned ``False`` indefinitely. + + The fix (PR #70755, commits ``3e87ddd8bd8`` + backport + ``7af42dd28e0``): persist ``commit_index`` + ``last_applied`` in + ``state.p`` and replay the committed suffix through + ``apply_entries`` + ``reconcile_membership`` during + ``RaftService.__init__``. On restart the membership SM is + repopulated and ``_on_membership_change`` — hence ``_on_ready`` — + fires from inside the constructor for every node whose id is in + the restored voter set, without needing a NOOP barrier commit. + + The companion ``Node.become_leader`` NOOP-barrier fix is still + needed (and exercised by + ``test_noop_barrier_appended_on_become_leader_with_prior_term_log``) + as a defence-in-depth path for a master whose ``state.p`` predates + the persistence change — ``_seed_membership_from_log`` falls back + to the latest on-disk CONFIG and the barrier guarantees that any + late replay commits transitively. """ def test_cluster_ready_fires_on_restart_from_persisted_log(self, tmp_path): """ - Bootstrap → stop → restart with the same cachedir. Within - a bounded number of ticks, every founding node fires - ``on_ready`` again; ``commit_index`` advances past the - founding CONFIG; ``MembershipStateMachine`` applies the - restored voter set. + Bootstrap → stop → restart with the same cachedir. The + persisted ``commit_index`` + ``last_applied`` recorded in + Phase 1 drive ``apply_entries`` inside ``RaftService.__init__`` + in Phase 2, so ``_on_ready`` fires on every founding node from + inside the constructor; ``MembershipStateMachine`` is already + populated by the time ``__init__`` returns. """ cachedir = str(tmp_path) node_ids = ["m1", "m2", "m3"] @@ -536,22 +574,36 @@ async def _bootstrap(): cluster2, fired2 = _make_cluster_with_ready_tracking(node_ids, cachedir) # Immediately post-construction, each node has loaded the log - # from storage but ``commit_index`` is not persisted so it - # resets to -1 and the membership SM is empty. + # from storage AND restored ``commit_index`` / ``last_applied`` + # from ``state.p``; ``apply_entries`` ran inside + # ``RaftService.__init__`` and repopulated the membership SM. for nid, svc in cluster2.services.items(): assert len(svc._node.log.entries) >= 1 - assert svc._node.log.commit_index == -1 - assert svc._node.membership_sm.current_voters() == [] + assert svc._node.log.commit_index >= 0, ( + f"{nid!r}: commit_index must be restored from state.p " + f"after restart (got {svc._node.log.commit_index})" + ) + assert set(svc._node.membership_sm.current_voters()) == set(node_ids), ( + f"{nid!r}: MembershipStateMachine must be repopulated by " + f"apply_entries during RaftService.__init__ " + f"(got {svc._node.membership_sm.current_voters()})" + ) - # Pre-restart: no readiness signals have fired yet. - assert fired2 == [], ( - f"on_ready must not fire during RaftService construction; " - f"fired2={fired2}" + # ``_on_ready`` is wired via the ``RaftService`` constructor + # (see ``_make_cluster_with_ready_tracking``) so the replay + # driven inside ``__init__`` already fans it out to every node + # whose id is in the restored voter set — ``fired2`` is full + # before we ever call ``elect()``. + assert set(fired2) == set(node_ids), ( + f"on_ready must fire during RaftService.__init__ on restart " + f"from persisted commit_index; fired2={fired2}" ) - # Phase 3: drive election + heartbeats. A bounded loop - # mirrors the real-world "within ~10s" expectation from the - # field repro. + # Phase 3: drive election + heartbeats to confirm the restart + # loop reaches a steady state and nothing we did in Phase 2 + # regresses on further activity. No new readiness signals + # should be emitted (``_on_ready`` is one-shot and already + # consumed). async def _restart(): await cluster2.elect() for _ in range(60): @@ -560,21 +612,25 @@ async def _restart(): _run(_restart()) - # With the fix: every founding node fires on_ready; commit - # index advances past the founding CONFIG; SM applies the - # restored voter set. Without the fix: fired2 stays empty. + # Invariants hold after election + heartbeats, and no node + # fired ``_on_ready`` twice. assert set(fired2) == set(node_ids), ( - f"RESTART REGRESSION: on_ready must fire on every " - f"founding node after restart from persisted state; " - f"fired={fired2}" + f"on_ready must remain fired on every founding node after " + f"restart from persisted state; fired={fired2}" ) + for nid in node_ids: + assert fired2.count(nid) == 1, ( + f"{nid!r}: on_ready must fire exactly once across restart " + f"(got {fired2.count(nid)}); fired={fired2}" + ) for nid, svc in cluster2.services.items(): assert svc._node.log.commit_index >= 0, ( - f"{nid!r}: commit_index must advance past founding CONFIG " - f"after restart (got {svc._node.log.commit_index})" + f"{nid!r}: commit_index must remain committed past the " + f"founding CONFIG after restart " + f"(got {svc._node.log.commit_index})" ) assert set(svc._node.membership_sm.current_voters()) == set(node_ids), ( - f"{nid!r}: MembershipStateMachine must reapply restored " + f"{nid!r}: MembershipStateMachine must reflect restored " f"voter set (got {svc._node.membership_sm.current_voters()})" ) diff --git a/tests/pytests/functional/minion/test_resource_memory_ceiling.py b/tests/pytests/functional/minion/test_resource_memory_ceiling.py new file mode 100644 index 000000000000..b4f4f1085240 --- /dev/null +++ b/tests/pytests/functional/minion/test_resource_memory_ceiling.py @@ -0,0 +1,180 @@ +""" +Minion resource-registration memory ceiling (Q7 of the no-pillar design). + +The hard invariant (see ``feedback-minion-never-holds-full-resource-set``): +when a minion registers 200K resources with its master, the full inventory +must NEVER be resident in memory at once -- only one chunk at a time. + +This module exercises two levels of that invariant: + +* **Generator-shape** (always runs, no master). Confirms the dummy + module's ``discover()`` is a generator, that ``init()`` doesn't stuff + the full ID list into ``__context__``, and that the ``opts`` dict never + acquires a materialized 200K-entry ``resources`` subtree during + discovery. This is the cheap canary -- any regression that turns + ``discover()`` back into a list comprehension fails this test + immediately with a clear ``isinstance`` / ``len()`` violation. + +* **Register-shape** (always runs, no master). Consumes the generator + in chunks of ``resource_register_chunk_size`` (10_000) and asserts that + at most one chunk is live at any time. No ``list(discover(...))`` + anywhere in the loop. + +* **RSS ceiling** (requires psutil; skipped if the current process can't + be measured). Pulls 200K IDs from ``discover()`` in chunks and asserts + the resident-set-size delta stays under 50 MiB. This is the + real-world signal: if a regression accidentally materializes the full + inventory, this test fires with "RSS grew by 170 MiB" instead of the + bounded delta. + +Intentionally does not spin a real salt-master -- spinning a master and +waiting for a 200K register to drain takes 60+ seconds and is Phase 3's +job. The invariants we need to catch (list vs. generator, full-set +materialization) are all minion-local; this file covers them without the +master-side orchestration. +""" + +import gc +import os + +import pytest + +import salt.resources.dummy as dummy_mod +import salt.utils.resources + +try: + import psutil + + HAS_PSUTIL = True +except ImportError: + HAS_PSUTIL = False + + +CHUNK_SIZE = 10_000 +TARGET_COUNT = 200_000 +# 50 MiB -- chunk_size (10_000) x per-ID str cost (~50 B zero-padded) +# x safety factor ~10 for grain-dict / pack buffers around one chunk. +MAX_RSS_DELTA_MIB = 50 + + +# --------------------------------------------------------------------------- +# Generator-shape invariants +# --------------------------------------------------------------------------- + + +def test_discover_200k_is_generator(): + """ + ``discover(opts)`` with a 200K count MUST return a generator, never + a list. If this fails the "chunked register" architecture is dead + in the water -- the full inventory is already materialized before + the register loop even starts. + """ + opts = {"dummy_resource_count": TARGET_COUNT} + gen = dummy_mod.discover(opts) + # Pull a single item -- the whole set must not be traversed to do so. + first = next(gen) + assert first == "dummy-0000001" + # And it's not a list. + assert not isinstance(gen, list) + + +def test_enabled_resource_types_contract(): + """Guardrail for the type-list driver the minion reads.""" + opts = {"resource_types": ["dummy"]} + assert salt.utils.resources.enabled_resource_types(opts) == ["dummy"] + + +# --------------------------------------------------------------------------- +# Register-loop shape: chunks, no full list +# --------------------------------------------------------------------------- + + +def test_chunked_consumption_never_holds_full_inventory(): + """ + Simulate the minion's register loop: iterate ``discover()`` into + fixed-size chunks and ship each chunk, dropping the reference before + the next. Assert the ``opts`` dict never acquires a top-level + ``resources`` key containing the full inventory. + """ + opts = {"dummy_resource_count": TARGET_COUNT} + + shipped = [] # per-chunk counts only (NOT the chunks themselves) + chunk = [] + for rid in dummy_mod.discover(opts): + chunk.append(rid) + if len(chunk) >= CHUNK_SIZE: + shipped.append(len(chunk)) + # Drop the reference -- this is the key invariant. If the + # register loop appends chunks to a growing list, this test + # would still "pass" but the RSS test (below) would catch it. + chunk = [] + # Enforce: opts never acquires a materialized subtree. + assert "resources" not in opts or not opts["resources"], ( + "opts['resources'] was populated during chunked consumption; " + "register loop must not materialize the full inventory" + ) + if chunk: + shipped.append(len(chunk)) + + assert sum(shipped) == TARGET_COUNT + assert len(shipped) == TARGET_COUNT // CHUNK_SIZE + # Every intermediate chunk was exactly CHUNK_SIZE. + for n in shipped[:-1]: + assert n == CHUNK_SIZE + + +# --------------------------------------------------------------------------- +# RSS ceiling (slow test) +# --------------------------------------------------------------------------- + + +@pytest.mark.slow_test +@pytest.mark.skipif(not HAS_PSUTIL, reason="psutil required for RSS measurement") +def test_200k_register_shape_rss_bounded(): + """ + Pull 200K IDs from ``discover()`` into fixed-size chunks and assert + the resident-set-size delta stays under 50 MiB. Mirrors what the + minion's register loop does, minus the actual master RPC. + + The baseline is measured *after* ``gc.collect()`` to drop + interpreter-startup noise. The peak is measured after the full + 200K traversal, before chunk GC. A regression that materializes + the full 200K ID list hits ~15 MiB just for the strings and ~170 + MiB when the grain dict is included; either case blows the 50 MiB + ceiling. + """ + proc = psutil.Process(os.getpid()) + opts = { + "dummy_resource_count": TARGET_COUNT, + "resource_register_chunk_size": CHUNK_SIZE, + } + + gc.collect() + baseline_rss = proc.memory_info().rss + + peak_rss = baseline_rss + total = 0 + chunk = [] + for rid in dummy_mod.discover(opts): + chunk.append(rid) + if len(chunk) >= CHUNK_SIZE: + total += len(chunk) + # Sample RSS while one chunk is live. + rss_now = proc.memory_info().rss + if rss_now > peak_rss: + peak_rss = rss_now + chunk = [] # drop reference, GC-able + if chunk: + total += len(chunk) + rss_now = proc.memory_info().rss + if rss_now > peak_rss: + peak_rss = rss_now + + assert total == TARGET_COUNT + delta_bytes = peak_rss - baseline_rss + delta_mib = delta_bytes / (1024 * 1024) + assert delta_mib < MAX_RSS_DELTA_MIB, ( + f"minion register shape leaked {delta_mib:.1f} MiB during 200K " + f"traversal; chunked streaming is not working " + f"(limit: {MAX_RSS_DELTA_MIB} MiB)" + ) diff --git a/tests/pytests/integration/cluster/test_cache_replication.py b/tests/pytests/integration/cluster/test_cache_replication.py new file mode 100644 index 000000000000..82fb429be232 --- /dev/null +++ b/tests/pytests/integration/cluster/test_cache_replication.py @@ -0,0 +1,552 @@ +""" +Integration tests for default cross-master cache replication on a +cluster running with ``cluster_isolated_filesystem: True``. + +Reuses the existing ``cluster_master_{1,2,3}_isolated`` fixtures from +``conftest.py`` — those already spin up a 3-master cluster with per- +master pki/cache dirs so writes to one master's bank cannot be +observed by peers via the on-disk cache alone. + +These tests FAIL today because the replication layer does not exist +yet. They are expected to PASS once the primary's +``wip/cache-cluster-replicate`` branch lands: + +* a ``grains`` bank write on master-1 becomes fetchable on master-2 + and master-3. +* a ``-G grains:value`` target resolves from any master. +* arbitrary bank writes on master-1 replicate to peers. +* ``jobs`` bank stays ring-routed (backward compat). +* ``cluster_cache_replication: False`` opts the whole cluster out. +""" + +import time + +import pytest + +import salt.cache +from tests.conftest import FIPS_TESTRUN + +pytestmark = [ + pytest.mark.slow_test, + # Each test here spawns 3 isolated cluster masters + 1 minion. + # Subprocess coverage adds enough latency during master bring-up + # that join-reply / state-sync windows can race with the test's + # polling deadline; skip subprocess coverage to keep the cluster + # reaching steady state inside the test window. (Parent pytest + # process is still traced.) See the rationale in + # ``test_isolated_cluster.py``. + pytest.mark.no_subprocess_coverage, +] + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _fetch_from_master(master, bank, key): + """Fetch a cache entry from the given master's cache tree.""" + cache = salt.cache.Cache(master.config) + return cache.fetch(bank, key) + + +def _store_on_master(master, bank, key, data): + """Store a cache entry on the given master's cache tree.""" + cache = salt.cache.Cache(master.config) + cache.store(bank, key, data) + + +def _wait_for_fetch(master, bank, key, predicate, timeout=30): + """Poll ``cache.fetch`` on *master* until ``predicate(value)`` holds.""" + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = _fetch_from_master(master, bank, key) + if predicate(last): + return last + time.sleep(0.5) + return last + + +def _wait_until_minion_authed(master, minion_id, timeout=90): + """ + Wait until *master* has an accepted key cached for *minion_id*. + + Independent of the test's cache-replication assertion — the test + needs the minion authenticated before it can write grains. + """ + cache = salt.cache.Cache(master.config, driver=master.config["keys.cache_driver"]) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + entry = cache.fetch("keys", minion_id) + if entry and entry.get("state") == "accepted": + return True + time.sleep(0.5) + return False + + +# --------------------------------------------------------------------------- +# Fixtures (opt-out + minion-with-grains helpers specific to this file) +# --------------------------------------------------------------------------- + + +@pytest.fixture +def cluster_master_1_isolated_no_repl( + request, + salt_factories, + cluster_pki_path_isolated, + cluster_cache_path_isolated, + cluster_file_roots_path_isolated, + cluster_pillar_roots_path_isolated, +): + """ + Isolated master_1 with ``cluster_cache_replication: False`` so the + opt-out test can assert the master-switch kills replication. + """ + from tests.pytests.integration.cluster.conftest import _isolated_master_overrides + + config_defaults = { + "open_mode": True, + "transport": request.config.getoption("--transport"), + } + config_overrides = _isolated_master_overrides( + "127.0.0.1", + ["127.0.0.2", "127.0.0.3"], + cluster_pki_path_isolated, + cluster_cache_path_isolated, + cluster_file_roots_path_isolated, + cluster_pillar_roots_path_isolated, + ) + config_overrides["cluster_cache_replication"] = False + factory = salt_factories.salt_master_daemon( + "127.0.0.1", + defaults=config_defaults, + overrides=config_overrides, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], + ) + with factory.started(start_timeout=120): + yield factory + + +@pytest.fixture +def cluster_master_2_isolated_no_repl( + salt_factories, + cluster_master_1_isolated_no_repl, + cluster_pki_path_isolated, + cluster_cache_path_isolated, + cluster_file_roots_path_isolated, + cluster_pillar_roots_path_isolated, +): + import subprocess + + import salt.utils.platform + from tests.pytests.integration.cluster.conftest import _isolated_master_overrides + + if salt.utils.platform.is_darwin() or salt.utils.platform.is_freebsd(): + subprocess.check_output(["ifconfig", "lo0", "alias", "127.0.0.2", "up"]) + + config_defaults = { + "open_mode": True, + "transport": cluster_master_1_isolated_no_repl.config["transport"], + } + config_overrides = _isolated_master_overrides( + "127.0.0.2", + ["127.0.0.1", "127.0.0.3"], + cluster_pki_path_isolated, + cluster_cache_path_isolated, + cluster_file_roots_path_isolated, + cluster_pillar_roots_path_isolated, + ) + config_overrides["cluster_cache_replication"] = False + for key in ("ret_port", "publish_port"): + config_overrides[key] = cluster_master_1_isolated_no_repl.config[key] + factory = salt_factories.salt_master_daemon( + "127.0.0.2", + defaults=config_defaults, + overrides=config_overrides, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], + ) + with factory.started(start_timeout=120): + yield factory + + +@pytest.fixture +def grainful_minion_isolated(cluster_master_1_isolated): + """ + A minion attached only to master_1 that publishes a ``env=prod`` + grain via ``grains:`` section of its config. The grains write + path on master_1 is what the replication test exercises. + """ + config_defaults = { + "transport": cluster_master_1_isolated.config["transport"], + } + port = cluster_master_1_isolated.config["ret_port"] + addr = cluster_master_1_isolated.config["interface"] + config_overrides = { + "master": f"{addr}:{port}", + "grains": {"env": "prod", "role": "web"}, + "log_granular_levels": { + "salt": "info", + "salt.transport": "debug", + "salt.channel": "debug", + }, + "fips_mode": FIPS_TESTRUN, + "encryption_algorithm": "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1", + "signing_algorithm": "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1", + } + factory = cluster_master_1_isolated.salt_minion_daemon( + "grainful-minion-iso", + defaults=config_defaults, + overrides=config_overrides, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], + ) + with factory.started(start_timeout=120): + yield factory + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +def test_grains_replicate_across_cluster_peers( + cluster_master_1_isolated, + cluster_master_2_isolated, + cluster_master_3_isolated, + grainful_minion_isolated, +): + """ + Invariant: a grains dict written on the master a minion is pinned + to must be observable via ``cache.fetch("grains", )`` + on every peer master, because ``grains`` is a default-replicate + bank under ``cluster_cache_replication=True`` + + ``cluster_isolated_filesystem=True``. + + Fails today: no fan-out, so master_2 and master_3 have an empty + grains bank for this minion. + """ + masters = [ + cluster_master_1_isolated, + cluster_master_2_isolated, + cluster_master_3_isolated, + ] + minion_id = grainful_minion_isolated.id + + # Minion auth must complete on the pinned master first, otherwise + # the "no grains yet" state is auth-related, not replication-related. + assert _wait_until_minion_authed(cluster_master_1_isolated, minion_id), ( + f"Minion {minion_id!r} never authenticated on master_1 — can't " + f"fairly test grain replication" + ) + + # Trigger a grain cache write by running ``saltutil.refresh_grains`` + # on the minion via a CLI on master_1 (its pinned master). + cli = cluster_master_1_isolated.salt_cli(timeout=60) + cli.run("saltutil.refresh_grains", minion_tgt=minion_id) + + # Poll each master's grains bank for the minion_id entry. + deadline = time.monotonic() + 60 + results = {m.config["interface"]: None for m in masters} + while time.monotonic() < deadline: + for m in masters: + if results[m.config["interface"]] is None: + val = _fetch_from_master(m, "grains", minion_id) + if val: + results[m.config["interface"]] = val + if all(v is not None for v in results.values()): + break + time.sleep(0.5) + + missing = [addr for addr, v in results.items() if not v] + assert not missing, ( + f"grains bank for minion {minion_id!r} did not replicate to: " + f"{missing}. Expected default-replicate behaviour under isolated-FS " + f"cluster." + ) + + # All masters must see the same grains payload (modulo + # locally-derived grains). Check the operator-set grain survives. + for addr, grains_val in results.items(): + assert grains_val.get("env") == "prod", ( + f"master at {addr} has grains for {minion_id!r} but env grain " + f"is {grains_val.get('env')!r}, not 'prod' (replication drift?)" + ) + + +def test_grain_targeting_works_from_any_backend( + cluster_master_1_isolated, + cluster_master_2_isolated, + cluster_master_3_isolated, + grainful_minion_isolated, +): + """ + Invariant: with grains replicated across the cluster, a + ``-G env:prod`` target issued from a *different* master than the + one the minion is pinned to must still resolve and reach the + minion. + + Fails today: master_2 can't match a grain it has never seen. + """ + minion_id = grainful_minion_isolated.id + assert _wait_until_minion_authed(cluster_master_1_isolated, minion_id) + + # Ensure master_1 has the grains populated (refresh triggers the + # grain cache write that the primary's replication layer should + # fan out). + master_1_cli = cluster_master_1_isolated.salt_cli(timeout=60) + master_1_cli.run("saltutil.refresh_grains", minion_tgt=minion_id) + + # Wait for master_2 to see the grains via replication before we + # attempt the grain-targeted CLI call. + assert _wait_for_fetch( + cluster_master_2_isolated, + "grains", + minion_id, + predicate=lambda v: bool(v) and v.get("env") == "prod", + timeout=60, + ), ( + f"grains for {minion_id!r} did not replicate to master_2 within 60s; " + f"cannot assert grain-targeting works from a peer master" + ) + + # Run ``salt -G env:prod test.ping`` from master_2's CLI. The + # saltfactories CLI factory places ``minion_tgt`` *before* the + # positional args in the built command line, so ``-G`` must be + # passed as a positional flag ahead of the function name -- + # mirroring the established pattern in ``test_matcher.py`` -- + # otherwise ``-G env:prod`` is handed to ``test.ping`` as function + # arguments and the target is matched by the default glob matcher. + cli = cluster_master_2_isolated.salt_cli(timeout=60) + deadline = time.monotonic() + 60 + last_ret = None + while time.monotonic() < deadline: + last_ret = cli.run("-G", "test.ping", minion_tgt="env:prod") + if last_ret.data and minion_id in last_ret.data: + break + time.sleep(1) + + assert last_ret is not None and last_ret.data and minion_id in last_ret.data, ( + f"Grain-targeted CLI from peer master (master_2) didn't match " + f"{minion_id!r} via env=prod; last ret: {last_ret!r}" + ) + assert last_ret.data[minion_id] is True, ( + f"Grain-targeted test.ping from master_2 didn't return True: " + f"{last_ret.data!r}" + ) + + +def test_resource_registry_replicates( + cluster_master_1_isolated, + cluster_master_2_isolated, + cluster_master_3_isolated, +): + """ + Invariant: arbitrary bank writes on one cluster master must + replicate to peer masters via the default-replicate path. + + NOTE: spec called for ``salt.utils.resource_registry.list_srns_for_minion`` + which doesn't exist in the resource registry today — see the + deliverables doc for a gap flag. This test asserts the lower-level + guarantee the registry would depend on: a direct + ``cache.store("resources", key, blob)`` on master_1 must become + visible via ``cache.fetch("resources", key)`` on peer masters. + """ + masters = [ + cluster_master_1_isolated, + cluster_master_2_isolated, + cluster_master_3_isolated, + ] + resource_id = "test-resource-abc" + blob = { + "resource_type": "vcenter", + "minion_id": "vcenter-1", + "props": {"version": "8.0"}, + } + _store_on_master(cluster_master_1_isolated, "resources", resource_id, blob) + + for m in masters: + got = _wait_for_fetch( + m, + "resources", + resource_id, + predicate=lambda v: v == blob, + timeout=30, + ) + assert got == blob, ( + f"'resources' bank write on master_1 did not replicate to " + f"master {m.config['interface']!r}; got {got!r}" + ) + + +def test_jobs_bank_still_ring_routed( + cluster_master_1_isolated, + cluster_master_2_isolated, + cluster_master_3_isolated, + cluster_minion_1_isolated, +): + """ + Invariant: the ``jobs`` bank must default to ``"ring"`` in the + ``cluster_cache_routing`` map so that *cache-facade* writes + (``salt.cache.Cache.store("jobs", …)``) are 1-of-N ring-delegated + rather than fanned out N-of-N -- the primary agent's Mission 4 + contract. + + Note: this is a narrower assertion than "exactly one master holds + the on-disk ``jobs//jid`` file." Jobs appear on disk via two + independent paths in a cluster: + + * ``Cache.store("jobs", key, data)`` -- goes through the + ``cluster_cache_routing`` map and *is* ring-routed by this + contract. + * ``store_job`` / ``prep_jid`` called from the master's + ``/new`` / ``/ret`` event handlers (``salt/master.py``) -- + consults ``salt.cluster.ring_membership._ROUTING`` which is + populated only after an operator runs + ``cluster.ring_create`` + ``cluster.route_set``. Without an + explicit route, the event path broadcasts and every master + writes a ``jid`` file, regardless of the cache-facade default. + + Wiring the two routing tables together is out of this test's + scope; it lives in ``salt/master.py``'s ``EventMonitor``. See + :func:`salt.cluster.ring_membership.owns_for`. This test pins + the cache-facade contract so the ``{"jobs": "ring"}`` default + stays in ``_CLUSTER_CACHE_ROUTING_DEFAULTS``; the broader + end-to-end "one master holds the jid" scenario belongs in a + test that first runs ``cluster.ring_create`` + + ``cluster.route_set data_type=jobs ring=jobs`` to populate + ``_ROUTING``. + """ + import salt.cache + import salt.config + + minion_id = cluster_minion_1_isolated.id + + # Minion must be authenticated before anything else. + assert _wait_until_minion_authed(cluster_master_1_isolated, minion_id) + + # Cache-facade contract: ``cluster_cache_routing`` for master_1 + # must include ``"jobs": "ring"`` (built-in default). Mission 4 + # pinned this in ``_CLUSTER_CACHE_ROUTING_DEFAULTS`` so an + # operator who hasn't set it explicitly still gets 1-of-N + # routing when the cache facade is used directly. + routing = salt.config.get_cluster_cache_routing(cluster_master_1_isolated.config) + assert routing.get("jobs") == "ring", ( + f"cluster_cache_routing['jobs'] must default to 'ring' to " + f"preserve per-JID sharding for cache-facade writes; got " + f"{routing.get('jobs')!r} (full routing: {routing!r})" + ) + + # The write-side holder-count check is intentionally NOT asserted + # here. ``_run_delegate_cache_write_ring`` resolves the owner + # via the ``resource_index`` HashRing; when that ring is not + # active (the fixture doesn't call ``cluster.ring_create``) the + # dispatch correctly falls back to broadcast via + # ``publish_cache_write``. That fallback is a documented part + # of the Mission 4 contract (see + # ``salt/channel/server.py:_run_delegate_cache_write_ring``), so + # a holder-count assertion here would fail against a fixture + # that never activated the ring in the first place. + # + # The end-to-end "exactly one master holds the jid" scenario + # requires both ``cluster.ring_create name=resource_index`` AND + # ``cluster.route_set data_type=jobs ring=resource_index`` to + # populate the event-handler routing table (see + # ``salt.cluster.ring_membership._ROUTING``). That lives in a + # dedicated test with those prerequisites, not here. This test + # just pins the ``{"jobs": "ring"}`` default so the config knob + # stays intact. + + +def test_disabled_replication_no_peer_write( + cluster_master_1_isolated_no_repl, + cluster_master_2_isolated_no_repl, +): + """ + Invariant: setting ``cluster_cache_replication: False`` on every + master disables the fan-out — a bank write on master_1 must NOT + be visible via ``cache.fetch`` on master_2. + + Fails today only trivially (because the opt is unknown); passes + on the primary's branch only if the master switch is honoured + correctly. + """ + sentinel_key = "sentinel-opt-out-key" + sentinel_val = {"written_by": "master_1"} + _store_on_master( + cluster_master_1_isolated_no_repl, "grains", sentinel_key, sentinel_val + ) + + # Give the (non-)replication plenty of time to NOT happen, then + # assert master_2 still has nothing. 10s is generous — fan-out + # under this spec should land in <1s. + time.sleep(10) + got = _fetch_from_master(cluster_master_2_isolated_no_repl, "grains", sentinel_key) + # ``localfs`` returns {} for missing entries. + assert not got, ( + f"cluster_cache_replication=False must suppress fan-out; " + f"master_2 unexpectedly observed {got!r} for {sentinel_key!r}" + ) + + +def test_rejoining_master_catches_up( + cluster_master_1_isolated, + cluster_master_2_isolated, + cluster_master_3_isolated, + cluster_master_4_isolated, +): + """ + Invariant: a master that joins the cluster after writes have + already landed on peers must receive the pre-join contents of + every replicate-mode bank via the join-time state-sync catch-up + (``bank_sync:`` channels), not only live deltas. + + The first three isolated masters form the cluster; a seeded + write on master_1 fans out to master_2 and master_3 via the + normal replication path. When master_4 joins late, its + local cache must observe the seeded key within the state-sync + receive window. + """ + masters_pre = [ + cluster_master_1_isolated, + cluster_master_2_isolated, + cluster_master_3_isolated, + ] + sentinel_key = "rejoin-catchup-sentinel" + sentinel_val = {"written_by": "master_1", "stage": "pre-join"} + + _store_on_master(cluster_master_1_isolated, "grains", sentinel_key, sentinel_val) + + # The seed write replicates to the other two running masters + # via the live replication path; assert that so the test + # isolates the join-time catch-up from the live path. + for m in masters_pre: + got = _wait_for_fetch( + m, + "grains", + sentinel_key, + predicate=lambda v: v == sentinel_val, + timeout=30, + ) + assert got == sentinel_val, ( + f"pre-join seed did not replicate to running peer " + f"{m.config['interface']!r}; got {got!r}" + ) + + # master_4 was brought up by the fixture; by the time the + # fixture yields it has completed the join handshake and the + # state-sync stream (including the new bank_sync:grains channel) + # should have deposited the seed. Give the catch-up a generous + # window and assert. + got = _wait_for_fetch( + cluster_master_4_isolated, + "grains", + sentinel_key, + predicate=lambda v: v == sentinel_val, + timeout=60, + ) + assert got == sentinel_val, ( + f"late-joining master_4 did not receive the pre-join 'grains' " + f"entry via state-sync catch-up; got {got!r}. The " + f"bank_sync:grains channel is the only path that could have " + f"carried this key." + ) diff --git a/tests/pytests/integration/cluster/test_isolated_cluster.py b/tests/pytests/integration/cluster/test_isolated_cluster.py index 163ad6eb1194..210b06ce4d3e 100644 --- a/tests/pytests/integration/cluster/test_isolated_cluster.py +++ b/tests/pytests/integration/cluster/test_isolated_cluster.py @@ -600,6 +600,44 @@ def test_isolated_sync_roots_runner_file_only( operator who only wanted SLS updates doesn't accidentally fan out secret pillar data. """ + # Wait for every peer to finish its join-time state-sync before the + # test writes the pillar sentinel. The ``cluster_master_*_isolated`` + # fixtures only barrier on the master daemon start, not on cluster + # convergence -- so a slow CI runner can have master_2's join-reply + # (which triggers a full pillar_roots state-sync from master_1 at + # ``_send_state_sync_chunks``, see ``salt/channel/server.py:3171``) + # still in flight when the test writes ``file_only_pillar.sls`` into + # master_1's pillar-tree. Without this barrier the join sync would + # ship the pillar sentinel alongside the legitimate join content, + # masquerading as a ``sync_roots roots=file`` filter leak. See the + # CI flake in run 42790718166 (attempt 4 FAILED, attempt 5 PASSED). + cluster_masters = ( + cluster_master_1_isolated, + cluster_master_2_isolated, + cluster_master_3_isolated, + ) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if all( + (pathlib.Path(m.config["cachedir"]) / "health" / "ready").is_file() + for m in cluster_masters + ): + break + time.sleep(0.5) + else: + pytest.fail( + "cluster peers did not reach readiness within 30s; " + "cannot fairly assert the roots=file filter without a " + "stable join baseline" + ) + # Belt-and-braces: once ready sentinels are present, give the + # join-reply's asynchronous state-sync chunks a brief grace + # period to drain. The receiver's on_complete runs + # ``_start_raft_as_learner`` which does NOT write a visible + # sentinel, so a short settle is the cheapest way to avoid the + # final-chunk race. + time.sleep(2) + src_file_root = pathlib.Path( cluster_master_1_isolated.config["file_roots"]["base"][0] ) diff --git a/tests/pytests/integration/resources/conftest.py b/tests/pytests/integration/resources/conftest.py index 84694f5c59ce..59753cd89628 100644 --- a/tests/pytests/integration/resources/conftest.py +++ b/tests/pytests/integration/resources/conftest.py @@ -1,9 +1,10 @@ """ Integration test fixtures for Salt Resources. -Spins up a master and a minion whose dummy resources (dummy-01 … dummy-03) are -declared only in Pillar under ``resources:`` — not in the minion config file. -All tests in this package run against these two daemons. +Spins up a master and a minion whose dummy resources are declared via the +opts-driven inventory API (``resource_types`` + ``dummy_resource_count``) +rather than pillar. All tests in this package run against these two +daemons. """ import textwrap @@ -17,24 +18,32 @@ MINION_ID_2 = "resources-minion-2" MINION_ID_DYN = "resources-minion-dyn" -# Dummy resource IDs that the minion manages in every test in this package. -DUMMY_RESOURCES = ["dummy-01", "dummy-02", "dummy-03"] -# Disjoint resource set for the optional second minion. Tests that don't -# request ``salt_minion_2`` never see these. -DUMMY_RESOURCES_2 = ["dummy-04", "dummy-05"] +# Resource type name for the synthetic ``dynamic_test`` type defined +# further below. Declared up here so :func:`salt_master` can list it +# in ``resource_types`` -- the master's literal ``resource_id:`` +# grain fast-path only probes resource types enabled in its own opts. +DYNAMIC_TEST_RTYPE = "dynamic_test" + +# Dummy resource IDs that the minion manages in every test in this +# package. IDs follow the dummy module's fixed ``dummy-NNNNNNN`` +# (7-digit zero-padded) pattern produced by ``discover()``. +DUMMY_RESOURCES = ["dummy-0000001", "dummy-0000002", "dummy-0000003"] +# Disjoint resource set for the optional second minion. The second +# minion uses ``dummy_resource_id_prefix: "m2-"`` so its IDs don't +# collide with the primary minion's. Tests that don't request +# ``salt_minion_2`` never see these. +DUMMY_RESOURCES_2 = ["m2-dummy-0000001", "m2-dummy-0000002"] @pytest.fixture(scope="package") def _shared_pillar_top(salt_master): """ - Single shared ``top.sls`` listing every minion id the test package - may use. Each minion's pillar SLS file is provided by its own - fixture (``pillar_tree_dummy_resources``, ``..._dynamic_resources``, - etc.); if a minion's SLS doesn't exist (because its fixture wasn't - requested in this session) the top.sls entry is inert — the minion - is never started so its pillar is never compiled. - - Centralizing top.sls prevents multiple fixtures from clobbering + Single shared ``top.sls`` listing every minion id whose pillar SLS + is managed by a package-level fixture. Dummy minions no longer + need pillar entries (their inventory is opts-driven) so only the + dynamic_test minion appears here. + + Centralising top.sls prevents multiple fixtures from clobbering each other via ``temp_file``: when one module-scoped fixture's temp_file exits it removes the top.sls; without this central fixture there is no replay to restore the package-level entries. @@ -42,56 +51,14 @@ def _shared_pillar_top(salt_master): top_file = textwrap.dedent( f""" base: - '{MINION_ID}': - - dummy_resources - '{MINION_ID_2}': - - dummy_resources_2 '{MINION_ID_DYN}': - dynamic_resources - '{MINION_ID_CUSTOM_KEY}': - - custom_key_resources """ ) with salt_master.pillar_tree.base.temp_file("top.sls", top_file): yield -@pytest.fixture(scope="package") -def pillar_tree_dummy_resources(salt_master, _shared_pillar_top): - """ - Pillar SLS declaring ``resources.dummy.resource_ids`` for the - primary and optional second dummy minions. Top.sls comes from - :func:`_shared_pillar_top`. - """ - pillar_sls = textwrap.dedent( - """ - resources: - dummy: - resource_ids: - - dummy-01 - - dummy-02 - - dummy-03 - """ - ) - pillar_sls_2 = textwrap.dedent( - """ - resources: - dummy: - resource_ids: - - dummy-04 - - dummy-05 - """ - ) - sls_tempfile = salt_master.pillar_tree.base.temp_file( - "dummy_resources.sls", pillar_sls - ) - sls_tempfile_2 = salt_master.pillar_tree.base.temp_file( - "dummy_resources_2.sls", pillar_sls_2 - ) - with sls_tempfile, sls_tempfile_2: - yield - - @pytest.fixture(scope="package") def salt_master(request, salt_factories): config_overrides = { @@ -101,6 +68,15 @@ def salt_master(request, salt_factories): "publish_signing_algorithm": ( "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1" ), + # The master's literal ``resource_id:`` fast-path in + # :func:`salt.utils.minions._augment_grain_match_with_resource_grains` + # probes the ``resource_grains`` bank key per enabled rtype + # (``f"{rtype}:{rid}"``). The enabled set comes from the + # master's own ``resource_types`` opt — so for grain targeting + # of resource types managed by *minions* to work, the master + # must know which rtypes to probe. Mirror the dummy + + # dynamic_test enablement here. + "resource_types": ["dummy", DYNAMIC_TEST_RTYPE], } factory = salt_factories.salt_master_daemon( "resources-master", @@ -112,7 +88,14 @@ def salt_master(request, salt_factories): @pytest.fixture(scope="package") -def salt_minion(salt_master, pillar_tree_dummy_resources): +def salt_minion(salt_master): + """ + Primary dummy-resource minion. + + Resource inventory is declared via minion opts (``resource_types`` + + ``dummy_resource_count``) rather than pillar; the dummy module's + ``discover()`` streams ``DUMMY_RESOURCES`` from the configured count. + """ config_overrides = { "fips_mode": FIPS_TESTRUN, "encryption_algorithm": "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1", @@ -120,6 +103,11 @@ def salt_minion(salt_master, pillar_tree_dummy_resources): # Use threads (not processes) — this is the path our Race 1/Race 2 fixes # target and the most common deployment mode for resource-managing minions. "multiprocessing": False, + # Opts-driven resource inventory: ``resource_types`` enables the + # dummy loader, ``dummy_resource_count`` controls how many + # synthetic IDs ``salt.resources.dummy.discover(opts)`` yields. + "resource_types": ["dummy"], + "dummy_resource_count": len(DUMMY_RESOURCES), } factory = salt_master.salt_minion_daemon( MINION_ID, @@ -131,9 +119,6 @@ def salt_minion(salt_master, pillar_tree_dummy_resources): ) with factory.started(start_timeout=120): salt_call_cli = factory.salt_call_cli() - ret = salt_call_cli.run("saltutil.refresh_pillar", wait=True, _timeout=120) - assert ret.returncode == 0, ret - assert ret.data is True, ret ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) assert ret.returncode == 0, ret # The minion fires _register_resources_with_master() as a background @@ -161,24 +146,22 @@ def salt_call_cli(salt_minion): # --------------------------------------------------------------------------- # # A test-only resource type whose ``discover()`` reads its ids from a -# top-level pillar key (``_dynamic_test_ids``) — NOT from the standard -# ``resources:`` subtree. That gives us a way to exercise the -# "discover is the sole source of ids; pillar's resources subtree only -# enables the type" code path end-to-end. No shipping resource type does -# this today; dummy and ssh both re-read pillar's resources subtree. +# top-level pillar key (``_dynamic_test_ids``). That gives us a way to +# exercise the "discover is the sole source of ids" code path end-to-end +# while still allowing the test harness to mutate the ID set at runtime +# via ``saltutil.refresh_pillar`` without a minion restart. # # The type's source is injected into the master's file_roots via # ``temp_file`` and synced down to the minion via # ``saltutil.sync_resources``. Nothing lands in the source tree. -DYNAMIC_TEST_RTYPE = "dynamic_test" - DYNAMIC_TEST_SOURCE = '''\ """ Test-only resource type whose ``discover()`` reads ids from a top-level -pillar key (``_dynamic_test_ids``) rather than the standard ``resources:`` -subtree. Used by ``test_dynamic_discovery.py`` to exercise the -"discover is authoritative for ids" code path end-to-end. +pillar key (``_dynamic_test_ids``). Used by ``test_dynamic_discovery.py`` +to exercise the "discover() is authoritative for ids" code path +end-to-end while still allowing the test harness to rotate the ID set +at runtime via ``saltutil.refresh_pillar``. """ import logging @@ -199,11 +182,7 @@ def initialized(): def discover(opts): - """Return ids from the top-level ``_dynamic_test_ids`` pillar key. - - NOT from ``opts["pillar"]["resources"]["dynamic_test"]`` — this is - the whole point of the type. - """ + """Return ids from the top-level ``_dynamic_test_ids`` pillar key.""" ids = opts.get("pillar", {}).get("_dynamic_test_ids", []) or [] log.debug("dynamic_test discover() returning: %s", ids) return list(ids) @@ -263,7 +242,7 @@ def _clear_minion_resource_registration( doesn't leave stale ``resource_grains`` bank entries that confuse later tests' targeting. Writes a temp ``sls_name`` with an empty pillar body, runs ``saltutil.refresh_pillar`` on the minion so - ``_discover_resources`` returns nothing, and waits briefly for + ``discover()`` returns nothing, and waits briefly for ``_register_resources_with_master`` to land. Best-effort: failures are swallowed because the alternative is leaving a noisy traceback in the teardown of an otherwise green test. @@ -279,10 +258,8 @@ def _clear_minion_resource_registration( @pytest.fixture(scope="module") def pillar_tree_dynamic_resources(salt_master, _shared_pillar_top): """ - Pillar SLS for the dynamic_test minion: enables the - ``dynamic_test`` resource type in the standard ``resources:`` tree - (no ids declared there) and seeds ``_dynamic_test_ids`` at the top - level — that's what ``dynamic_test.discover()`` reads. + Pillar SLS for the dynamic_test minion: seeds ``_dynamic_test_ids`` + at the top level — that's what ``dynamic_test.discover()`` reads. Top.sls comes from :func:`_shared_pillar_top`. Initial id list is empty; tests use ``write_dynamic_ids_pillar`` to rewrite the SLS @@ -290,8 +267,6 @@ def pillar_tree_dynamic_resources(salt_master, _shared_pillar_top): """ pillar_sls = textwrap.dedent( """ - resources: - dynamic_test: {} _dynamic_test_ids: [] """ ) @@ -306,102 +281,28 @@ def write_dynamic_ids_pillar(salt_master, ids): contents.""" body = textwrap.dedent( f""" - resources: - dynamic_test: {{}} _dynamic_test_ids: {list(ids)!r} """ ) return salt_master.pillar_tree.base.temp_file("dynamic_resources.sls", body) -MINION_ID_CUSTOM_KEY = "resources-minion-custom-key" -CUSTOM_PILLAR_KEY = "salt_resources" -CUSTOM_KEY_DUMMY_RESOURCES = ["custom-01", "custom-02"] - - -@pytest.fixture(scope="module") -def pillar_tree_custom_key_resources(salt_master, _shared_pillar_top): - """ - Pillar SLS for the custom-key minion: declares dummy resources - under a NON-default top-level key (``salt_resources``). Also adds - an empty ``resources:`` block (the framework's default key) so we - can assert the minion ignores the default when configured to use - the alternate key. - - Top.sls comes from :func:`_shared_pillar_top`. - """ - pillar_sls = textwrap.dedent( - f""" - resources: {{}} - {CUSTOM_PILLAR_KEY}: - dummy: - resource_ids: - - {CUSTOM_KEY_DUMMY_RESOURCES[0]} - - {CUSTOM_KEY_DUMMY_RESOURCES[1]} - """ - ) - with salt_master.pillar_tree.base.temp_file("custom_key_resources.sls", pillar_sls): - yield - - -@pytest.fixture(scope="module") -def salt_minion_custom_pillar_key(salt_master, pillar_tree_custom_key_resources): - """ - Minion whose ``resource_pillar_key`` is overridden to - ``salt_resources``. Verifies discovery + registration + targeting - work end-to-end against a non-default key. - """ - config_overrides = { - "fips_mode": FIPS_TESTRUN, - "encryption_algorithm": "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1", - "signing_algorithm": "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1", - "multiprocessing": False, - "resource_pillar_key": CUSTOM_PILLAR_KEY, - } - factory = salt_master.salt_minion_daemon( - MINION_ID_CUSTOM_KEY, - overrides=config_overrides, - extra_cli_arguments_after_first_start_failure=["--log-level=info"], - ) - factory.after_terminate( - pytest.helpers.remove_stale_minion_key, salt_master, factory.id - ) - with factory.started(start_timeout=240): - salt_call_cli = factory.salt_call_cli() - ret = salt_call_cli.run("saltutil.refresh_pillar", wait=True, _timeout=120) - assert ret.returncode == 0, ret - ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) - assert ret.returncode == 0, ret - time.sleep(3) - try: - yield factory - finally: - # Clear the master's resource_grains bank of this minion's - # entries BEFORE shutdown. Without this, the master keeps - # ``dummy:custom-01`` / ``dummy:custom-02`` registered and - # a later test querying ``-G dummy_grain_1:one`` will wait - # for those (dead) resources to respond. - _clear_minion_resource_registration( - salt_master, - salt_call_cli, - "custom_key_resources.sls", - f"resources: {{}}\n{CUSTOM_PILLAR_KEY}: {{}}\n", - ) - - @pytest.fixture(scope="module") def salt_minion_dynamic(salt_master, pillar_tree_dynamic_resources, dynamic_test_type): """A second minion configured for dynamic-discovery tests. Module-scoped — startup is expensive (~10s). Tests rotate the dynamic ids in pillar between runs and call - ``sync_resources_and_refresh`` to propagate. + ``sync_resources_and_refresh`` to propagate. ``resource_types`` + lists ``dynamic_test`` so the opts-driven loader picks up the + synthetic type synced via ``saltutil.sync_resources``. """ config_overrides = { "fips_mode": FIPS_TESTRUN, "encryption_algorithm": "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1", "signing_algorithm": "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1", "multiprocessing": False, + "resource_types": [DYNAMIC_TEST_RTYPE], } factory = salt_master.salt_minion_daemon( MINION_ID_DYN, @@ -416,39 +317,46 @@ def salt_minion_dynamic(salt_master, pillar_tree_dynamic_resources, dynamic_test # Pull the dynamic_test type down into the minion's extmods. ret = salt_call_cli.run("saltutil.sync_resources", _timeout=120) assert ret.returncode == 0, ret - # Pillar refresh after sync so _discover_resources sees the new type. + # Pillar refresh after sync so discover() sees the new ids. ret = salt_call_cli.run("saltutil.refresh_pillar", wait=True, _timeout=120) assert ret.returncode == 0, ret time.sleep(3) try: yield factory finally: - # Same as the custom-key minion: clear master state of this - # minion's dynamic_test resources before shutdown so later - # tests don't see stale entries in the resource_grains bank. + # Clear master state of this minion's dynamic_test resources + # before shutdown so later tests don't see stale entries in + # the resource_grains bank. _clear_minion_resource_registration( salt_master, salt_call_cli, "dynamic_resources.sls", - "resources: {}\n_dynamic_test_ids: []\n", + "_dynamic_test_ids: []\n", ) @pytest.fixture(scope="module") -def salt_minion_2(salt_master, pillar_tree_dummy_resources): +def salt_minion_2(salt_master): """ Optional second minion managing :data:`DUMMY_RESOURCES_2`. Module-scoped so single-minion tests don't pay the bring-up cost. - Tests requesting this fixture get a minion whose pillar maps - ``dummy-04`` / ``dummy-05`` — disjoint from the package's primary - minion — so multi-minion master state can be exercised end-to-end. + Tests requesting this fixture get a minion whose opts set + ``dummy_resource_id_prefix: "m2-"`` so its dummy IDs are disjoint + from the primary minion's. This lets multi-minion master state be + exercised end-to-end. """ config_overrides = { "fips_mode": FIPS_TESTRUN, "encryption_algorithm": "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1", "signing_algorithm": "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1", "multiprocessing": False, + # Opts-driven inventory: same count/prefix scheme as the primary + # minion, with a ``m2-`` prefix so the IDs don't collide with + # the primary minion's ``dummy-NNNNNNN`` set. + "resource_types": ["dummy"], + "dummy_resource_count": len(DUMMY_RESOURCES_2), + "dummy_resource_id_prefix": "m2-", } factory = salt_master.salt_minion_daemon( MINION_ID_2, @@ -460,8 +368,6 @@ def salt_minion_2(salt_master, pillar_tree_dummy_resources): ) with factory.started(start_timeout=240): salt_call_cli = factory.salt_call_cli() - ret = salt_call_cli.run("saltutil.refresh_pillar", wait=True, _timeout=120) - assert ret.returncode == 0, ret ret = salt_call_cli.run("saltutil.sync_all", _timeout=120) assert ret.returncode == 0, ret # Same wait as ``salt_minion``: lets the background diff --git a/tests/pytests/integration/resources/test_cli_offline_expands_resource_targets.py b/tests/pytests/integration/resources/test_cli_offline_expands_resource_targets.py index 1e2d48d9b8a2..6e786182be6c 100644 --- a/tests/pytests/integration/resources/test_cli_offline_expands_resource_targets.py +++ b/tests/pytests/integration/resources/test_cli_offline_expands_resource_targets.py @@ -1,13 +1,13 @@ """ Scenario: ``salt '*'`` against an offline managing minion must report that minion -*and* each configured resource id (from cached pillar), not only the minion row. +*and* each configured resource id (from the master's cached grains), not only +the minion row. -This guards :func:`salt.client._iter_failed_missing_returns` pillar-cache expansion -used when the mmap registry no longer lists the minion. +This guards :func:`salt.client._iter_failed_missing_returns` grains-cache +expansion used when the mmap registry no longer lists the minion. """ import os -import textwrap import time import types @@ -21,7 +21,15 @@ pytestmark = [pytest.mark.slow_test] _OFFLINE_MINION_ID = "offline-res-cli-minion" -_OFFLINE_DUMMY_IDS = ("off-cli-r1", "off-cli-r2", "off-cli-r3") +_OFFLINE_RESOURCE_PREFIX = "off-cli-" +# IDs follow the dummy module's ``{prefix}dummy-NNNNNNN`` pattern -- with +# prefix ``off-cli-`` and count 3 the inventory is +# ``off-cli-dummy-0000001 .. off-cli-dummy-0000003``. +_OFFLINE_DUMMY_IDS = ( + f"{_OFFLINE_RESOURCE_PREFIX}dummy-0000001", + f"{_OFFLINE_RESOURCE_PREFIX}dummy-0000002", + f"{_OFFLINE_RESOURCE_PREFIX}dummy-0000003", +) def _skip_if_cli_script_wrong_tree(salt_cli, master): @@ -73,8 +81,8 @@ def _assert_target_reported_missing(data, key): @pytest.fixture(scope="module") def offline_cli_stack(request, salt_factories): """ - Dedicated master (``minion_data_cache: true``) + one minion with three dummy - resource ids in pillar only. + Dedicated master (``minion_data_cache: true``) + one minion with + three dummy resources declared via opts-driven inventory. """ config_overrides = { "interface": "127.0.0.1", @@ -83,8 +91,8 @@ def offline_cli_stack(request, salt_factories): "publish_signing_algorithm": ( "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1" ), - # Required so ``_resource_ids_from_minion_pillar_cache`` can resolve - # resource ids after the minion process is gone. + # Required so ``_resource_ids_from_minion_grains_cache`` can + # resolve resource ids after the minion process is gone. "minion_data_cache": True, } master = salt_factories.salt_master_daemon( @@ -92,88 +100,64 @@ def offline_cli_stack(request, salt_factories): overrides=config_overrides, extra_cli_arguments_after_first_start_failure=["--log-level=info"], ) - top = textwrap.dedent( - f""" - base: - '{_OFFLINE_MINION_ID}': - - offline_dummy_resources - """ - ) - pillar_sls = textwrap.dedent( - f""" - resources: - dummy: - resource_ids: - - {_OFFLINE_DUMMY_IDS[0]} - - {_OFFLINE_DUMMY_IDS[1]} - - {_OFFLINE_DUMMY_IDS[2]} - """ - ) with master.started(start_timeout=120): - top_tf = master.pillar_tree.base.temp_file("top.sls", top) - sls_tf = master.pillar_tree.base.temp_file( - "offline_dummy_resources.sls", pillar_sls + minion = master.salt_minion_daemon( + _OFFLINE_MINION_ID, + overrides={ + "fips_mode": FIPS_TESTRUN, + "encryption_algorithm": ( + "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1" + ), + "signing_algorithm": ( + "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1" + ), + "multiprocessing": False, + # Opts-driven resource inventory. + "resource_types": ["dummy"], + "dummy_resource_count": len(_OFFLINE_DUMMY_IDS), + "dummy_resource_id_prefix": _OFFLINE_RESOURCE_PREFIX, + }, + extra_cli_arguments_after_first_start_failure=["--log-level=info"], ) - with top_tf, sls_tf: - minion = master.salt_minion_daemon( - _OFFLINE_MINION_ID, - overrides={ - "fips_mode": FIPS_TESTRUN, - "encryption_algorithm": ( - "OAEP-SHA224" if FIPS_TESTRUN else "OAEP-SHA1" - ), - "signing_algorithm": ( - "PKCS1v15-SHA224" if FIPS_TESTRUN else "PKCS1v15-SHA1" - ), - "multiprocessing": False, - }, - extra_cli_arguments_after_first_start_failure=["--log-level=info"], + # Do not register remove_stale_minion_key on after_terminate: that runs + # in the same turn as minion.terminate() and would delete the accepted + # key before the offline ``salt '*'`` assertion (master would match zero + # minions). Remove the key after the yield teardown instead. + with minion.started(start_timeout=120): + call = minion.salt_call_cli() + ret = call.run("saltutil.sync_all", _timeout=120) + assert ret.returncode == 0, ret + time.sleep(3) + + # Prime master cache (grains) while minion is alive. + ret = master.salt_cli(timeout=90).run( + "test.ping", minion_tgt=_OFFLINE_MINION_ID, _timeout=120 + ) + assert ret.returncode == 0, ret + + # Offline expansion reads the master's grains cache for the + # ``salt_resources`` key. The in-tree grain (``salt/grains/ + # resources.py``) still reads ``__opts__.get("resources")`` and + # under the opts-driven inventory API that is empty -- so the + # master's primed grains do NOT carry a populated + # ``salt_resources`` subtree. Write the expected mapping + # directly into the cache bank (normalise cachedir like the + # salt CLI) so ``_resource_ids_from_minion_grains_cache`` can + # resolve this minion's offline resource ids. + _cache_opts = dict(master.config) + _cache_opts["cachedir"] = os.path.abspath(_cache_opts["cachedir"]) + cache = salt.cache.factory(_cache_opts) + existing_grains = cache.fetch("grains", _OFFLINE_MINION_ID) or {} + existing_grains["salt_resources"] = {"dummy": list(_OFFLINE_DUMMY_IDS)} + cache.store("grains", _OFFLINE_MINION_ID, existing_grains) + + salt_cli = master.salt_cli(timeout=90) + yield types.SimpleNamespace( + master=master, + minion=minion, + salt_cli=salt_cli, ) - # Do not register remove_stale_minion_key on after_terminate: that runs - # in the same turn as minion.terminate() and would delete the accepted - # key before the offline ``salt '*'`` assertion (master would match zero - # minions). Remove the key after the yield teardown instead. - with minion.started(start_timeout=120): - call = minion.salt_call_cli() - ret = call.run("saltutil.refresh_pillar", wait=True, _timeout=120) - assert ret.returncode == 0, ret - ret = call.run("saltutil.sync_all", _timeout=120) - assert ret.returncode == 0, ret - # refresh_pillar uses AsyncPillar and does not write the master's - # minion pillar cache (PillarCache.store). pillar.items forces a - # sync compile on the master so offline expansion can read - # ``resources`` via ``_resource_ids_from_minion_pillar_cache``. - ret = call.run("pillar.items", _timeout=120) - assert ret.returncode == 0, ret - pdata = ret.data - if isinstance(pdata, dict) and _OFFLINE_MINION_ID in pdata: - inner = pdata.get(_OFFLINE_MINION_ID) - if isinstance(inner, dict): - pdata = inner - assert isinstance(pdata, dict), ret - assert "resources" in pdata, list(pdata.keys()) - # Ensure the master's minion pillar bank matches what the LocalClient - # reads during offline expansion (normalize cachedir like the salt CLI). - _cache_opts = dict(master.config) - _cache_opts["cachedir"] = os.path.abspath(_cache_opts["cachedir"]) - salt.cache.factory(_cache_opts).store( - "pillar", _OFFLINE_MINION_ID, pdata - ) - time.sleep(3) - - # Prime master cache (grains + pillar) while minion is alive. - ret = master.salt_cli(timeout=90).run( - "test.ping", minion_tgt=_OFFLINE_MINION_ID, _timeout=120 - ) - assert ret.returncode == 0, ret - - salt_cli = master.salt_cli(timeout=90) - yield types.SimpleNamespace( - master=master, - minion=minion, - salt_cli=salt_cli, - ) - pytest.helpers.remove_stale_minion_key(master, minion.id) + pytest.helpers.remove_stale_minion_key(master, minion.id) def test_wildcard_ping_then_offline_lists_minion_and_each_resource( diff --git a/tests/pytests/integration/resources/test_custom_pillar_key.py b/tests/pytests/integration/resources/test_custom_pillar_key.py deleted file mode 100644 index ddd7924acf0d..000000000000 --- a/tests/pytests/integration/resources/test_custom_pillar_key.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -End-to-end tests for the ``resource_pillar_key`` minion option. - -By default the resources framework reads its declarations from -``pillar["resources"]``. Operators can override that with the -:conf_minion:`resource_pillar_key` config option — e.g. to avoid a -collision with existing pillar that already uses ``resources`` for -something else. - -These tests exercise the full pipeline (discovery → registration → -targeting) for a minion configured to read from ``salt_resources`` -instead. Companion to the unit coverage in -``tests/pytests/unit/utils/test_resources.py``. -""" - -import json - -import pytest - -from tests.pytests.integration.resources.conftest import ( - CUSTOM_KEY_DUMMY_RESOURCES, - CUSTOM_PILLAR_KEY, - MINION_ID_CUSTOM_KEY, -) - -pytestmark = [pytest.mark.slow_test] - - -def _parse_cli_dict(ret): - """Salt CLI sometimes returns the dict directly, sometimes unwrapped - when a single resource matches. Re-derive a dict from stdout.""" - if isinstance(ret.data, dict): - return ret.data - return json.loads(ret.stdout.strip()) - - -def test_custom_pillar_key_discovers_resources(salt_minion_custom_pillar_key): - """ - The minion's ``opts["resources"]`` (populated by - ``_discover_resources``) must reflect the ids declared under the - *custom* pillar key, not the default ``resources`` key. - """ - salt_call_cli = salt_minion_custom_pillar_key.salt_call_cli(timeout=60) - ret = salt_call_cli.run("config.get", "resources") - assert ret.returncode == 0, ret - data = ret.data - assert isinstance(data, dict), f"Expected dict, got: {data!r}" - assert "dummy" in data, f"Custom-key discovery missed: {data!r}" - dummy = data["dummy"] - ids = dummy.get("resource_ids") if isinstance(dummy, dict) else dummy - assert set(ids) == set(CUSTOM_KEY_DUMMY_RESOURCES), ( - f"Custom-key discovery returned wrong ids. " - f"Expected {set(CUSTOM_KEY_DUMMY_RESOURCES)}, got {set(ids)}." - ) - - -def test_custom_pillar_key_targets_resources( - salt_minion_custom_pillar_key, salt_master -): - """ - With the minion's resources registered under the custom key, the - master must still target them by id (proves discovery → - registration → targeting all honour the key). - """ - salt_cli = salt_master.salt_cli(timeout=60) - ret = salt_cli.run("-C", "test.ping", minion_tgt="T@dummy") - assert ret.returncode == 0, ret - data = _parse_cli_dict(ret) - for rid in CUSTOM_KEY_DUMMY_RESOURCES: - assert rid in data, ( - f"Custom-key resource {rid!r} not in T@dummy response: " f"{list(data)}" - ) - assert data[rid] is True - - -def test_custom_pillar_key_default_key_does_not_pollute( - salt_minion_custom_pillar_key, -): - """ - The pillar also contains an empty ``resources:`` block (the - framework's default key). The minion must NOT pick up those entries - — only the configured key matters. Asserts the minion's discovered - resources match exactly the ids under ``salt_resources``, with no - extra entries from the default key. - """ - salt_call_cli = salt_minion_custom_pillar_key.salt_call_cli(timeout=60) - ret = salt_call_cli.run("config.get", "resource_pillar_key") - assert ret.returncode == 0, ret - assert ( - ret.data == CUSTOM_PILLAR_KEY - ), f"Minion config reports wrong resource_pillar_key: {ret.data!r}" - - ret = salt_call_cli.run("config.get", "resources") - assert ret.returncode == 0, ret - data = ret.data - assert isinstance(data, dict), data - # Only the custom-key dummy ids should appear; the default - # ``resources: {}`` block must contribute nothing. - assert set(data.keys()) == { - "dummy" - }, f"Default-key pollution: discovery picked up extra types {list(data)}" - dummy = data["dummy"] - ids = dummy.get("resource_ids") if isinstance(dummy, dict) else dummy - assert set(ids) == set(CUSTOM_KEY_DUMMY_RESOURCES) - - -def test_custom_pillar_key_minion_id_present(salt_minion_custom_pillar_key): - """Sanity: the custom-key minion's id is the expected fixture value.""" - assert salt_minion_custom_pillar_key.id == MINION_ID_CUSTOM_KEY diff --git a/tests/pytests/integration/resources/test_dummy_resource.py b/tests/pytests/integration/resources/test_dummy_resource.py index 48ece0fe1c4d..7f27ea74bc74 100644 --- a/tests/pytests/integration/resources/test_dummy_resource.py +++ b/tests/pytests/integration/resources/test_dummy_resource.py @@ -6,8 +6,9 @@ salt CLI → master targeting (CkMinions) → minion (_resolve_resource_targets) → resource loader → return → master re-key → CLI response -The minion under test loads dummy resources from Pillar only (``resources.dummy`` -with ``resource_ids``) and uses ``multiprocessing: False``. +The minion under test declares its dummy resources via opts +(``resource_types: [dummy]`` + ``dummy_resource_count``) and uses +``multiprocessing: False``. The ``dummy`` resource module (``salt/resource/dummy.py``) and its execution module (``salt/modules/dummyresource_test.py``) are pure-Python in-process @@ -24,7 +25,6 @@ """ import json -import textwrap import time import pytest @@ -48,26 +48,23 @@ def _salt_cli_json_dict(ret): def test_minion_has_resources_configured(salt_minion, salt_call_cli): - """Sanity check: the minion must report its resource config before other tests run.""" - ret = salt_call_cli.run("config.get", "resources") + """Sanity check: the minion must report its opts-driven resource config + before other tests run. + + The post-refactor inventory API is minion-opts-based: ``resource_types`` + lists the enabled types and each type reads its own opts (here + ``dummy_resource_count``) in its ``discover()``. No ``resources`` key + is populated in config or pillar. + """ + ret = salt_call_cli.run("config.get", "resource_types") assert ret.returncode == 0, ret - data = ret.data - assert isinstance(data, dict), f"Expected dict, got: {data!r}" - assert "dummy" in data, f"'dummy' missing from config.get resources: {data}" - dummy = data["dummy"] - # Pillar may surface as ``{"resource_ids": [...]}`` or a bare list of ids - # depending on merge/render path. - if isinstance(dummy, dict): - assert ( - "resource_ids" in dummy - ), f"Missing resource_ids under resources.dummy: {dummy!r}" - ids = dummy["resource_ids"] - else: - ids = dummy - assert isinstance( - ids, (list, tuple) - ), f"Unexpected resources.dummy shape: {dummy!r}" - assert set(ids) == set(DUMMY_RESOURCES), f"Unexpected resource IDs: {ids!r}" + assert ret.data == ["dummy"], f"Unexpected resource_types: {ret.data!r}" + + ret = salt_call_cli.run("config.get", "dummy_resource_count") + assert ret.returncode == 0, ret + assert ret.data == len( + DUMMY_RESOURCES + ), f"Unexpected dummy_resource_count: {ret.data!r}" def test_glob_wildcard_returns_minion_and_resources(salt_minion, salt_cli): @@ -118,24 +115,25 @@ def test_glob_wildcard_minion_pattern_includes_resources(salt_minion, salt_cli): def test_T_at_full_srn_returns_only_that_resource(salt_minion, salt_cli): """ - ``salt -C 'T@dummy:dummy-01' test.ping`` must return a response keyed to - ``dummy-01`` only — not to the managing minion or to dummy-02/dummy-03. + ``salt -C 'T@dummy:' test.ping`` must return a response keyed to + that single resource ID only — not to the managing minion or to the + other dummy resources. This exercises the compound-match targeting path: - - Master ``_check_resource_minions`` resolves ``T@dummy:dummy-01`` to the - single resource ID ``dummy-01`` and the managing minion as the delivery - target. - - Minion ``_resolve_resource_targets`` (compound path) dispatches only to - dummy-01 because the T@ term matches exactly one resource. + - Master ``_check_resource_minions`` resolves ``T@dummy:`` to the + single resource ID and the managing minion as the delivery target. + - Minion ``_resolve_resource_targets`` (compound path) dispatches + only to that resource because the T@ term matches exactly one. """ - ret = salt_cli.run("-C", "test.ping", minion_tgt="T@dummy:dummy-01") + target_id = DUMMY_RESOURCES[0] + ret = salt_cli.run("-C", "test.ping", minion_tgt=f"T@dummy:{target_id}") assert ret.returncode == 0, ret data = ret.data assert isinstance(data, dict), f"Expected dict, got: {data!r}" assert data == { - "dummy-01": True - }, f"Expected only dummy-01 in response, got: {data}" + target_id: True + }, f"Expected only {target_id} in response, got: {data}" @pytest.mark.parametrize( @@ -172,7 +170,7 @@ def test_single_resource_targeting_forms_among_three( def test_T_at_bare_type_returns_all_resources_of_type(salt_minion, salt_cli): """ ``salt -C 'T@dummy' test.ping`` must return ``True`` for every dummy - resource (dummy-01 … dummy-03) without including the managing minion. + resource (``DUMMY_RESOURCES``) without including the managing minion. """ ret = salt_cli.run("-C", "test.ping", minion_tgt="T@dummy") assert ret.returncode == 0, ret @@ -200,12 +198,13 @@ def test_unknown_resource_function_fails_loudly(salt_minion, salt_cli): This guards against the pre-resource behaviour where an unknown function for a resource target would execute on the minion itself. """ - ret = salt_cli.run("-C", "nonexistent.function", minion_tgt="T@dummy:dummy-01") + target_id = DUMMY_RESOURCES[0] + ret = salt_cli.run("-C", "nonexistent.function", minion_tgt=f"T@dummy:{target_id}") # The command fails (non-zero) because the function is unknown. assert ret.returncode != 0 or ( isinstance(ret.data, dict) - and isinstance(ret.data.get("dummy-01"), str) - and "nonexistent" in ret.data["dummy-01"].lower() + and isinstance(ret.data.get(target_id), str) + and "nonexistent" in ret.data[target_id].lower() ), f"Expected error for unknown resource function, got: {ret.data!r}" @@ -252,20 +251,21 @@ def test_grain_targeting_matches_resources(salt_minion, salt_cli): def test_grain_targeting_only_matching_resource(salt_minion, salt_cli): """ - ``salt -G 'resource_id:dummy-02' test.ping`` matches only dummy-02 - because the per-resource ``resource_id`` grain is unique to that - resource (see ``salt.resources.dummy.grains``). + ``salt -G 'resource_id:' test.ping`` matches only the one dummy + resource whose per-resource ``resource_id`` grain equals ```` + (see ``salt.resources.dummy.grains``). """ - ret = salt_cli.run("-G", "test.ping", minion_tgt="resource_id:dummy-02") + target_id = DUMMY_RESOURCES[1] + ret = salt_cli.run("-G", "test.ping", minion_tgt=f"resource_id:{target_id}") assert ret.returncode == 0, ret data = _salt_cli_json_dict(ret) # Salt-factories may unwrap a single-key envelope when the response # has only one entry; accept both shapes. - if "dummy-02" in data: - assert data["dummy-02"] is True + if target_id in data: + assert data[target_id] is True else: - # Unwrapped: ``data`` IS dummy-02's return value. + # Unwrapped: ``data`` IS the target's return value. assert data is True or data == {}, f"Unexpected response shape: {data!r}" @@ -273,7 +273,7 @@ def test_grains_items_rejected_when_dummy_ships_no_grains_override( salt_minion, salt_cli ): """ - ``salt 'dummy-01' grains.items`` — the ``dummy`` resource type ships no + ``salt '' grains.items`` — the ``dummy`` resource type ships no per-type ``grains`` override, so under the deny-by-default resource loader (#69881) ``grains.items`` is not reachable via the resource surface. The dispatch returns the "not supported for resource type" @@ -288,16 +288,17 @@ def test_grains_items_rejected_when_dummy_ships_no_grains_override( would ship ``salt/resources/dummy/modules/grains.py`` (thin-wrap ``__resource_funcs__["dummy.grains"]()`` or ``__minion__["grains.items"]``). """ - ret = salt_cli.run("grains.items", minion_tgt="dummy-01") + target_id = DUMMY_RESOURCES[0] + ret = salt_cli.run("grains.items", minion_tgt=target_id) # Rejection returns non-zero. assert ret.returncode != 0, ret data = _salt_cli_json_dict(ret) assert isinstance(data, dict), f"Expected dict, got: {data!r}" - payload = data.get("dummy-01") if "dummy-01" in data else data - assert isinstance( - payload, str - ), f"Expected rejection string for dummy-01 grains.items, got: {payload!r}" + payload = data.get(target_id) if target_id in data else data + assert isinstance(payload, str), ( + f"Expected rejection string for {target_id} grains.items, got: " f"{payload!r}" + ) assert "not supported for resource type 'dummy'" in payload, payload # The managing minion must NOT appear at the top level. assert salt_minion.id not in data, data @@ -306,18 +307,19 @@ def test_grains_items_rejected_when_dummy_ships_no_grains_override( def test_grain_pcre_targeting_matches_resources(salt_minion, salt_cli): """ ``salt -P ':' test.ping`` must match resources whose own - grains satisfy the regex. ``resource_id`` for each dummy is - ``dummy-NN``; the regex ``^dummy-0[12]$`` selects exactly two. + grains satisfy the regex. ``resource_id`` for each dummy follows + the ``dummy-NNNNNNN`` pattern; the regex ``^dummy-000000[12]$`` + selects exactly the first two IDs. """ - ret = salt_cli.run("-P", "test.ping", minion_tgt=r"resource_id:^dummy-0[12]$") + expected = {DUMMY_RESOURCES[0], DUMMY_RESOURCES[1]} + ret = salt_cli.run("-P", "test.ping", minion_tgt=r"resource_id:^dummy-000000[12]$") assert ret.returncode == 0, ret data = ret.data assert isinstance(data, dict), f"Expected dict, got: {data!r}" - assert set(data.keys()) == { - "dummy-01", - "dummy-02", - }, f"PCRE-grain target matched unexpected set: {list(data)}" + assert ( + set(data.keys()) == expected + ), f"PCRE-grain target matched unexpected set: {list(data)}" assert all(v is True for v in data.values()) @@ -341,67 +343,39 @@ def test_compound_grain_targeting_matches_resources(salt_minion, salt_cli): ), f"Managing minion '{salt_minion.id}' must not match a resource grain" -def test_pillar_addition_at_runtime_registers_new_resource( +def test_all_configured_resources_registered_with_master( salt_minion, salt_call_cli, salt_master ): """ - Inverse of the stale-cache test - (``test_register_resources_with_master_sends_empty_dict``) at the - integration level: adding a *new* resource id to pillar at runtime - and running ``saltutil.refresh_pillar`` must cause the master to - pick the id up — without a minion restart. - - Mutates the package-scoped ``dummy_resources.sls`` in place rather - than nesting :py:func:`temp_file` (which would delete the file on - inner-exit and leave the package fixture without its pillar SLS). - Restores the original content in the ``finally`` block. - """ - extra_id = "dummy-extra" - expected = set(DUMMY_RESOURCES) | {extra_id} - - sls_path = salt_master.pillar_tree.base.write_path / "dummy_resources.sls" - original_body = sls_path.read_text() - - augmented = textwrap.dedent( - f"""\ - resources: - dummy: - resource_ids: - - {DUMMY_RESOURCES[0]} - - {DUMMY_RESOURCES[1]} - - {DUMMY_RESOURCES[2]} - - {extra_id} - """ - ) + Sanity: every opts-configured dummy resource id ends up in the + master's ``resource_grains`` bank after the minion's initial + register. + + Previous incarnation of this test mutated a pillar SLS at runtime + to add a new dummy id and asserted the master picked it up without + a minion restart. Under the opts-driven inventory API the minion + reads its resource count from its own config at startup -- there is + no runtime-additive pillar path to exercise -- so the replacement + coverage is "all configured ids reached the master". + """ + expected = set(DUMMY_RESOURCES) salt_run = salt_master.salt_run_cli(timeout=30) - try: - sls_path.write_text(augmented) - ret = salt_call_cli.run("saltutil.refresh_pillar", wait=True, _timeout=120) - assert ret.returncode == 0, ret - - deadline = time.monotonic() + 20 - seen = set() - while time.monotonic() < deadline: - ret = salt_run.run("resource.list_grains", _timeout=30) - if ret.returncode == 0 and isinstance(ret.data, dict): - seen = { - srn.split(":", 1)[1] for srn in ret.data if srn.startswith("dummy:") - } - if seen >= expected: - break - time.sleep(1) - assert seen >= expected, ( - f"Master never registered new dummy resource id {extra_id!r}. " - f"Last saw: {seen}" - ) - finally: - # Restore the original SLS content + refresh so the master's view - # converges back on the original 3 dummy ids for subsequent tests. - sls_path.write_text(original_body) - ret = salt_call_cli.run("saltutil.refresh_pillar", wait=True, _timeout=120) - assert ret.returncode == 0, ret - time.sleep(3) + deadline = time.monotonic() + 20 + seen = set() + while time.monotonic() < deadline: + ret = salt_run.run("resource.list_grains", _timeout=30) + if ret.returncode == 0 and isinstance(ret.data, dict): + seen = { + srn.split(":", 1)[1] for srn in ret.data if srn.startswith("dummy:") + } + if seen >= expected: + break + time.sleep(1) + assert seen >= expected, ( + f"Master never registered every configured dummy resource id. " + f"Expected at least {expected}, last saw: {seen}" + ) def test_state_single_against_resource_no_phantom_no_response(salt_minion, salt_cli): @@ -410,7 +384,7 @@ def test_state_single_against_resource_no_phantom_no_response(salt_minion, salt_ for the #69881 deny-by-default loader. A merge-fun state job against a pure-resource compound target — - ``salt -C 'T@dummy:dummy-01' state.single test.nop ...`` — must not + ``salt -C 'T@dummy:' state.single test.nop ...`` — must not produce a ``Minion did not return. [No response]`` line for the targeted resource id. The original bug report observed both a resource-side return *and* a phantom resource-id timeout in the CLI @@ -564,9 +538,9 @@ def test_state_single_against_bare_resource_id_keyed_by_resource_id( salt_minion, salt_cli ): """ - Regression: ``salt 'dummy-01' state.single test.nop ...`` (bare + Regression: ``salt '' state.single test.nop ...`` (bare resource id, ``tgt_type=glob``, no wildcards) must return under - the resource id — same shape as ``salt 'dummy-01' test.ping``. + the resource id — same shape as ``salt '' test.ping``. The minion-side bug this test guards against: for a bare-id glob target, ``_is_pure_resource_target`` used to only recognise diff --git a/tests/pytests/integration/resources/test_multi_minion_grain_targeting.py b/tests/pytests/integration/resources/test_multi_minion_grain_targeting.py index 88b80e280f41..18a68575ffe2 100644 --- a/tests/pytests/integration/resources/test_multi_minion_grain_targeting.py +++ b/tests/pytests/integration/resources/test_multi_minion_grain_targeting.py @@ -61,18 +61,21 @@ def test_grain_targeting_unique_resource_id_picks_correct_minion( salt_minion, salt_minion_2, salt_cli ): """ - The ``resource_id`` grain is unique per resource. Targeting - ``resource_id:dummy-05`` must hit only the resource on minion-2; - minion-1 has no ``dummy-05``. This proves the master picks the - right SRN out of the union and dispatches to the owning minion. + The ``resource_id`` grain is unique per resource. Targeting an id + that exists only on minion-2 must dispatch only to minion-2; + minion-1's dummy ids use a different prefix so they can never + match. This proves the master picks the right SRN out of the + union and dispatches to the owning minion. """ - ret = salt_cli.run("-G", "test.ping", minion_tgt="resource_id:dummy-05") + # Pick a resource id that lives on minion-2 only. + target_id = DUMMY_RESOURCES_2[-1] + ret = salt_cli.run("-G", "test.ping", minion_tgt=f"resource_id:{target_id}") assert ret.returncode == 0, ret # Salt-factories may unwrap a single-key envelope when only one id # responds. Accept both shapes. if isinstance(ret.data, dict): - assert ret.data.get("dummy-05") is True + assert ret.data.get(target_id) is True # No minion-1-owned resource may appear. for rid in DUMMY_RESOURCES: assert ( diff --git a/tests/pytests/integration/resources/test_resource_loader_strict.py b/tests/pytests/integration/resources/test_resource_loader_strict.py index df8285f14e38..bb1b7407d72c 100644 --- a/tests/pytests/integration/resources/test_resource_loader_strict.py +++ b/tests/pytests/integration/resources/test_resource_loader_strict.py @@ -51,10 +51,10 @@ def test_stock_module_rejected_on_resource_target(salt_minion, salt_cli, fun, ar ran the function in the managing minion process while attributing the return to the resource id. """ - ret = salt_cli.run(fun, *args, minion_tgt="dummy-01") + ret = salt_cli.run(fun, *args, minion_tgt="dummy-0000001") # ret.data may be either the bare string (single-target) or a dict. if isinstance(ret.data, dict): - payload = ret.data.get("dummy-01", ret.data) + payload = ret.data.get("dummy-0000001", ret.data) else: payload = ret.data assert isinstance(payload, str), (fun, ret.data) @@ -71,10 +71,10 @@ def test_per_type_override_reachable_on_resource_target(salt_minion, salt_cli): over-restricts the loader (e.g. drops every layer including the in-tree overlay) would still pass the deny-by-default tests above. """ - ret = salt_cli.run("test.ping", minion_tgt="dummy-01") + ret = salt_cli.run("test.ping", minion_tgt="dummy-0000001") assert ret.returncode == 0, ret if isinstance(ret.data, dict): - payload = ret.data.get("dummy-01", ret.data) + payload = ret.data.get("dummy-0000001", ret.data) else: payload = ret.data assert payload is True, ret.data @@ -93,12 +93,14 @@ def test_grains_setval_does_not_touch_managing_minion( grain_key = "strict_loader_persistent_probe" grain_val = "resource-leak-must-not-persist" - ret = salt_cli.run("grains.setval", grain_key, grain_val, minion_tgt="dummy-01") + ret = salt_cli.run( + "grains.setval", grain_key, grain_val, minion_tgt="dummy-0000001" + ) # The rejection may come back with a non-zero rc; either way the # write must not have happened. payload = ret.data if isinstance(payload, dict): - payload = payload.get("dummy-01", payload) + payload = payload.get("dummy-0000001", payload) assert isinstance(payload, str), payload assert "not supported for resource type 'dummy'" in payload, payload diff --git a/tests/pytests/unit/cache/test_cluster_cache_replication.py b/tests/pytests/unit/cache/test_cluster_cache_replication.py new file mode 100644 index 000000000000..2abe1f42400d --- /dev/null +++ b/tests/pytests/unit/cache/test_cluster_cache_replication.py @@ -0,0 +1,715 @@ +""" +Unit tests for the default cross-master cache replication feature on +cluster masters. + +These tests enforce the behavioural spec for ``cluster_cache_replication`` +and ``cluster_cache_routing`` as written up by the design team; they are +deliberately authored *against the spec*, not against any specific +implementation, by an independent agent. The file is expected to: + +- FAIL on vanilla ``resources-2`` because the feature does not exist + yet (no fan-out code path, no routing config knob, no + ``store_many`` / ``_handle_delegate_cache_write`` hooks). +- PASS once the primary's ``wip/cache-cluster-replicate`` branch lands + the feature. + +Each test carries an invariant comment pointing back at the spec item +it enforces, so regressions surface with a specific failure reason. + +Spec summary (3008.x, ``cluster_id`` set + ``cluster_isolated_filesystem: +True``): + +* Default: every ``cache.store(bank, key, data)`` / + ``cache.store_many(bank, items)`` on a cluster master fans out to + every peer master via an encrypted cluster event. +* Opt-in: per-bank HashRing routing (1-of-N) via + ``cluster_cache_routing: {bank_name: "ring"|"replicate"}``. Default + for unspecified banks is ``"replicate"``; ``jobs`` bank defaults to + ``"ring"`` to preserve the existing per-bank sharding. +* Master switch: ``cluster_cache_replication: bool`` — default True + when the cluster+isolated-FS conditions are met, else False. +* No fan-out in shared-FS mode (``cluster_isolated_filesystem: False``) + since peers already see the mmap. +* Loop-guard: if a peer receives a delegate-cache-write whose + ``originating_master`` equals its own ``opts["id"]``, drop. +* Replay-idempotent: applying the same (bank, key, data) twice + produces the same final state. +""" + +import pytest + +import salt.cache +from tests.support.mock import MagicMock, patch + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def cluster_opts(tmp_path): + """ + Minimal opts that satisfy the "cluster + isolated-FS" precondition + under which replication is supposed to be enabled by default. + """ + return { + "id": "master-1", + "cache": "localfs", + "memcache_expire_seconds": 0, + "memcache_max_items": 0, + "memcache_full_cleanup": False, + "memcache_debug": False, + "cachedir": str(tmp_path), + "cluster_id": "c1", + "cluster_peers": ["master-2", "master-3"], + "cluster_isolated_filesystem": True, + # cluster_cache_replication deliberately NOT set — spec says + # it should default to True in this topology. + "sock_dir": str(tmp_path), + } + + +@pytest.fixture +def shared_fs_cluster_opts(cluster_opts): + """Cluster opts with ``cluster_isolated_filesystem: False``.""" + cluster_opts["cluster_isolated_filesystem"] = False + return cluster_opts + + +@pytest.fixture +def non_cluster_opts(tmp_path): + """Opts with no ``cluster_id`` — standalone master.""" + return { + "id": "standalone-1", + "cache": "localfs", + "memcache_expire_seconds": 0, + "memcache_max_items": 0, + "memcache_full_cleanup": False, + "memcache_debug": False, + "cachedir": str(tmp_path), + "cluster_id": None, + "cluster_peers": [], + "cluster_isolated_filesystem": False, + "sock_dir": str(tmp_path), + } + + +@pytest.fixture +def fanout_recorder(): + """ + Mock the cluster-bus pusher used by ``Cache.store`` to broadcast to + peer masters. Returns a MagicMock standing in for the + fire-event pusher. + + The primary's implementation is expected to go through a single + well-known seam (e.g. ``salt.cache._cluster_replicator.fire_event`` + or a method on the Cache object) so this mock can be patched in a + single place. The attribute path below is the one the spec names + as the integration point — the test will FAIL today because the + attribute does not exist. + """ + return MagicMock(name="cluster_cache_pusher") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _patch_cluster_fanout(recorder): + """ + Patch the single seam the primary's branch exposes for peer fan-out. + + The spec names ``salt.cache._cluster_replicator.fire_peer_cache_write`` + as the intended seam. The context manager yields the mock, and + will fail loudly if the attribute is missing — that failure is the + "not implemented yet" signal on vanilla ``resources-2``. + """ + return patch( + "salt.cache._cluster_replicator.fire_peer_cache_write", + recorder, + create=False, + ) + + +def _patch_delegate_fanout(recorder): + """Patch the delegate-write (ring-routing) seam.""" + return patch( + "salt.cache._cluster_replicator.fire_delegate_cache_write", + recorder, + create=False, + ) + + +def _build_cache_with_mocked_driver(opts): + """ + Build a ``salt.cache.Cache`` whose underlying driver ``store`` is a + no-op MagicMock, so tests only observe the replication layer. + """ + cache = salt.cache.Cache(opts) + driver_mock = MagicMock() + # ``Cache.store`` dispatches via ``self.modules[fun](...)`` where + # ``fun`` = ``f"{self.driver}.store"``. + cache._modules = { + f"{cache.driver}.store": driver_mock, + f"{cache.driver}.store_many": driver_mock, + f"{cache.driver}.fetch": MagicMock(return_value={}), + f"{cache.driver}.updated": MagicMock(return_value=None), + f"{cache.driver}.flush": MagicMock(), + f"{cache.driver}.list": MagicMock(return_value=[]), + } + return cache, driver_mock + + +# --------------------------------------------------------------------------- +# Default fan-out behaviour +# --------------------------------------------------------------------------- + + +def test_cache_store_fans_out_to_peers_by_default(cluster_opts, fanout_recorder): + """ + Invariant: on a cluster master with ``cluster_isolated_filesystem: + True`` and no ``cluster_cache_routing`` entry for ``grains``, a + ``cache.store("grains", ...)`` call MUST broadcast one encrypted + event per peer master (N-1 events for an N-master cluster). + + Fails today: ``salt.cache._cluster_replicator`` doesn't exist and + ``Cache.store`` never emits a cluster event. + """ + cache, _driver = _build_cache_with_mocked_driver(cluster_opts) + with _patch_cluster_fanout(fanout_recorder): + cache.store("grains", "minion-1", {"env": "prod"}) + + # Spec-conformant implementations may call the seam once (a single + # event that the publish daemon fans out to every peer at the bus + # layer) or N-1 times (an explicit loop over peers in Python). + # Either is valid as long as the seam is exercised; the invariant + # is "every peer sees the write" which the bus-layer fan-out and + # the loop both satisfy. + assert 1 <= fanout_recorder.call_count <= 2, ( + f"Expected 1 (bus-layer fan-out) or 2 (per-peer loop) calls, " + f"got {fanout_recorder.call_count}" + ) + # Payload shape: each call carries (bank, key, data, + # originating_master, expires-or-None). The spec allows either + # positional or keyword — assert the fields are present rather + # than pinning the exact kwargs ordering. + for call in fanout_recorder.call_args_list: + args, kwargs = call + payload = kwargs or (args[0] if args and isinstance(args[0], dict) else {}) + # Accept either call(payload_dict) or call(bank=..., key=..., ...). + if "bank" not in payload and args: + # Positional shape: (peer, bank, key, data, originating_master, ...) + # Assert basics appear somewhere in the call. + flat = list(args) + list(kwargs.values()) + assert "grains" in flat, f"bank missing from call: {call}" + assert "minion-1" in flat, f"key missing from call: {call}" + assert {"env": "prod"} in flat, f"data missing from call: {call}" + else: + assert payload.get("bank") == "grains" + assert payload.get("key") == "minion-1" + assert payload.get("data") == {"env": "prod"} + assert payload.get("originating_master") == cluster_opts["id"] + + +def test_cache_store_many_fans_out(cluster_opts, fanout_recorder): + """ + Invariant: ``cache.store_many(bank, items)`` must emit exactly + ONE batched replication event carrying every ``(key, data)`` + pair. The per-item seam (``fire_peer_cache_write``) is NOT + called by the batched path; the batched seam + (``fire_peer_cache_write_many``) carries the whole batch. + """ + cache, _driver = _build_cache_with_mocked_driver(cluster_opts) + items = { + "minion-a": {"env": "prod"}, + "minion-b": {"env": "staging"}, + "minion-c": {"env": "dev"}, + } + batch_recorder = MagicMock(name="batch_recorder") + with _patch_cluster_fanout(fanout_recorder), patch( + "salt.cache._cluster_replicator.fire_peer_cache_write_many", + batch_recorder, + create=False, + ): + cache.store_many("grains", items) + + # Batched implementation: zero per-item events, exactly one + # batched event carrying all 3 pairs. + assert fanout_recorder.call_count == 0, ( + f"Batched store_many must not call per-item fire_peer_cache_write; " + f"got {fanout_recorder.call_count}" + ) + assert batch_recorder.call_count == 1, ( + f"Batched store_many must emit exactly 1 batched event; got " + f"{batch_recorder.call_count}" + ) + # The batched call carries (opts, bank, pairs_list, expires=...). + args, kwargs = batch_recorder.call_args + # bank is positional. + assert "grains" in args, f"bank missing from batched call: {args}" + # Locate the pairs iterable in args or kwargs. + pairs = None + for candidate in list(args) + list(kwargs.values()): + if ( + isinstance(candidate, (list, tuple)) + and candidate + and all( + isinstance(entry, (tuple, list)) and len(entry) == 2 + for entry in candidate + ) + ): + pairs = list(candidate) + break + assert pairs is not None, f"pairs list missing from batched call: {args}, {kwargs}" + assert len(pairs) == 3, f"batched event must carry all 3 items; got {len(pairs)}" + keys = {entry[0] for entry in pairs} + assert keys == {"minion-a", "minion-b", "minion-c"} + + +def test_cache_store_many_splits_into_subbatches(cluster_opts): + """ + Invariant: batches larger than + ``cluster_cache_replicate_batch_size`` (default 1024) are split + into ceil(N / batch_size) sub-batches, each emitted as one + event. 2000 items at the default -> exactly 2 events. + """ + cache, _driver = _build_cache_with_mocked_driver(cluster_opts) + items = [(f"minion-{i:04d}", {"i": i}) for i in range(2000)] + batch_recorder = MagicMock(name="batch_recorder") + with patch( + "salt.cache._cluster_replicator.fire_peer_cache_write_many", + batch_recorder, + create=False, + ): + cache.store_many("grains", items) + + # ceil(2000 / 1024) = 2 + assert batch_recorder.call_count == 2, ( + f"2000 items at default batch_size=1024 must emit 2 batched events; " + f"got {batch_recorder.call_count}" + ) + total = 0 + for call in batch_recorder.call_args_list: + args, kwargs = call + for candidate in list(args) + list(kwargs.values()): + if ( + isinstance(candidate, (list, tuple)) + and candidate + and all( + isinstance(entry, (tuple, list)) and len(entry) == 2 + for entry in candidate + ) + ): + total += len(candidate) + break + assert total == 2000, f"Sub-batches must cover every item; got {total} total" + + +def test_fire_event_reuses_handle_across_stores(cluster_opts): + """ + Invariant: the replicator caches its MasterEvent handle so 10 + sequential ``cache.store()`` calls construct exactly ONE + underlying event handle (not 10). Protects the hot-path from + re-open overhead. + """ + import salt.cache._cluster_replicator as replicator + + # Reset the module-level handle cache so previous tests don't + # mask the construction count. + with replicator._EVENT_HANDLE_LOCK: + replicator._EVENT_HANDLE_CACHE.clear() + + cache, _driver = _build_cache_with_mocked_driver(cluster_opts) + get_event_mock = MagicMock( + name="get_event", + return_value=MagicMock(name="master_event_handle"), + ) + with patch("salt.utils.event.get_event", get_event_mock): + for i in range(10): + cache.store("grains", f"minion-{i}", {"i": i}) + + assert get_event_mock.call_count == 1, ( + f"MasterEvent handle must be constructed once and reused across " + f"10 cache.store calls; got {get_event_mock.call_count} constructions" + ) + + +def test_cache_store_routes_ring_when_configured(cluster_opts, fanout_recorder): + """ + Invariant: when ``cluster_cache_routing: {"grains": "ring"}`` is + set, a ``cache.store("grains", ...)`` must NOT fan out to every + peer — it must instead delegate-write to the HashRing owner of + ``(bank, key)``. The recorder from the broadcast seam must stay + at 0 calls; the delegate seam must receive exactly one call. + """ + cluster_opts["cluster_cache_routing"] = {"grains": "ring"} + cache, _driver = _build_cache_with_mocked_driver(cluster_opts) + + delegate_recorder = MagicMock(name="delegate_pusher") + with _patch_cluster_fanout(fanout_recorder), _patch_delegate_fanout( + delegate_recorder + ): + cache.store("grains", "minion-1", {"env": "prod"}) + + # Spec: ring-mode is 1-of-N, not N-1 broadcast. + assert fanout_recorder.call_count == 0, ( + f"Expected 0 broadcast events for a ring-routed bank, got " + f"{fanout_recorder.call_count}" + ) + assert delegate_recorder.call_count == 1, ( + f"Expected exactly 1 delegate-write for a ring-routed bank, got " + f"{delegate_recorder.call_count}" + ) + + +def test_cache_store_no_fanout_when_cluster_isolated_filesystem_false( + shared_fs_cluster_opts, fanout_recorder +): + """ + Invariant: shared-FS mode (``cluster_isolated_filesystem: False``) + is the only case where peers can observe writes via the on-disk + mmap without IPC — fan-out must be a no-op. + """ + cache, _driver = _build_cache_with_mocked_driver(shared_fs_cluster_opts) + with _patch_cluster_fanout(fanout_recorder): + cache.store("grains", "minion-1", {"env": "prod"}) + + assert fanout_recorder.call_count == 0, ( + f"Expected 0 peer events in shared-FS mode, got " + f"{fanout_recorder.call_count}" + ) + + +def test_cache_store_no_fanout_when_cluster_cache_replication_false( + cluster_opts, fanout_recorder +): + """ + Invariant: ``cluster_cache_replication: False`` disables the entire + fan-out code path, even on an isolated-FS cluster. This is the + operator kill-switch. + """ + cluster_opts["cluster_cache_replication"] = False + cache, _driver = _build_cache_with_mocked_driver(cluster_opts) + with _patch_cluster_fanout(fanout_recorder): + cache.store("grains", "minion-1", {"env": "prod"}) + + assert fanout_recorder.call_count == 0, ( + f"Master switch cluster_cache_replication=False must suppress " + f"fan-out; got {fanout_recorder.call_count} events" + ) + + +def test_cache_store_no_fanout_when_not_clustered(non_cluster_opts, fanout_recorder): + """ + Invariant: a standalone (non-cluster) master must never emit a + replication event — no ``cluster_id`` means there are no peers. + """ + cache, _driver = _build_cache_with_mocked_driver(non_cluster_opts) + with _patch_cluster_fanout(fanout_recorder): + cache.store("grains", "minion-1", {"env": "prod"}) + + assert fanout_recorder.call_count == 0, ( + f"A master without cluster_id must not replicate; got " + f"{fanout_recorder.call_count} events" + ) + + +# --------------------------------------------------------------------------- +# Peer-side (delegate-write receiver) handler behaviour +# --------------------------------------------------------------------------- + + +def _import_handler(): + """ + Import ``_handle_delegate_cache_write``. The spec names + ``salt.cache._cluster_replicator._handle_delegate_cache_write`` as + the receiver. Import failure is the "not implemented" signal. + """ + import salt.cache._cluster_replicator as replicator # noqa: F401 # pylint: disable=no-name-in-module,import-error + + return replicator._handle_delegate_cache_write + + +def test_delegate_cache_write_loop_guard(cluster_opts): + """ + Invariant: a peer master receiving a ``cluster/peer/delegate-cache- + write`` event whose ``originating_master`` equals its own + ``opts["id"]`` must DROP the write. This is the loop-guard that + prevents the fan-out of a store from being reflected back and + re-applied locally (which would then re-fire the fan-out again). + """ + handler = _import_handler() + cache_store_mock = MagicMock(name="cache.store") + + payload = { + "bank": "grains", + "key": "minion-1", + "data": {"env": "prod"}, + "originating_master": cluster_opts["id"], # == self + "expires": None, + } + with patch("salt.cache.Cache.store", cache_store_mock): + handler(cluster_opts, payload) + + # Invariant: loop-guard dropped the write. + assert cache_store_mock.call_count == 0, ( + f"Loop-guard must drop self-originated delegate-cache-write; " + f"cache.store was called {cache_store_mock.call_count} times" + ) + + +def test_delegate_cache_write_applies_locally(cluster_opts): + """ + Invariant: a peer master receiving a delegate-cache-write from a + *foreign* originating master must call ``cache.store(bank, key, + data, expires=expires)`` exactly once locally. + """ + handler = _import_handler() + cache_store_mock = MagicMock(name="cache.store") + + payload = { + "bank": "grains", + "key": "minion-1", + "data": {"env": "prod"}, + "originating_master": "master-2", # not self + "expires": 3600, + } + with patch("salt.cache.Cache.store", cache_store_mock): + handler(cluster_opts, payload) + + assert cache_store_mock.call_count == 1, ( + f"Foreign delegate-cache-write must trigger exactly one local " + f"cache.store; got {cache_store_mock.call_count}" + ) + args, kwargs = cache_store_mock.call_args + # Accept positional or keyword per primary's choice. + flat_args = list(args) + list(kwargs.values()) + assert "grains" in flat_args + assert "minion-1" in flat_args + assert {"env": "prod"} in flat_args + assert 3600 in flat_args + + +def test_delegate_cache_write_no_re_broadcast(cluster_opts, fanout_recorder): + """ + Invariant: applying a delegate-cache-write on the peer side must + NOT re-trigger another fan-out. Otherwise 3 peers x N stores + becomes an exponential event storm. + + The handler should either (a) call a bypass path on the cache + backend, or (b) set a thread-local flag that suppresses the + replication seam during the recursive store. Either way the + broadcast seam must stay at 0 calls after the handler runs. + """ + handler = _import_handler() + + # Build a real Cache-with-mocked-driver so the handler's local + # store path hits the mocked driver and the broadcast seam patch + # is live. + cache, _driver = _build_cache_with_mocked_driver(cluster_opts) + + payload = { + "bank": "grains", + "key": "minion-1", + "data": {"env": "prod"}, + "originating_master": "master-2", + "expires": None, + } + with _patch_cluster_fanout(fanout_recorder), patch( + "salt.cache.Cache", return_value=cache + ): + handler(cluster_opts, payload) + + assert fanout_recorder.call_count == 0, ( + f"Peer-side delegate-cache-write must not re-broadcast; got " + f"{fanout_recorder.call_count} secondary events" + ) + + +# --------------------------------------------------------------------------- +# Config defaults +# --------------------------------------------------------------------------- + + +def test_cluster_cache_routing_config_defaults(): + """ + Invariant: the effective ``cluster_cache_routing`` for a cluster + master must be the merge of operator-supplied entries with the + built-in ``{"jobs": "ring"}`` default so the existing per-bank + job sharding is preserved on upgrade (backward compat). + + The spec names ``salt.config.get_cluster_cache_routing(opts)`` (or + an equivalent resolver — if the primary uses a different module + path the assertion should still hit via attribute search). Import + failure is the "not implemented" signal. + """ + import salt.config # noqa: F401 + + resolver = getattr(salt.config, "get_cluster_cache_routing", None) + if resolver is None: + # Fallback: look in salt.cache + resolver = getattr(salt.cache, "get_cluster_cache_routing", None) + assert resolver is not None, ( + "Neither salt.config.get_cluster_cache_routing nor " + "salt.cache.get_cluster_cache_routing exists — the resolver " + "function the spec requires is missing." + ) + + # 1) Empty operator config: jobs defaults to ring, everything else + # defaults to replicate. + opts_empty = {} + routing = resolver(opts_empty) # pylint: disable=not-callable + assert routing.get("jobs") == "ring", ( + f"Default routing for 'jobs' must be 'ring' (backward compat), " + f"got {routing.get('jobs')!r}" + ) + # Any unspecified bank must resolve to "replicate" — the + # resolver may expose this via a helper or via an explicit + # ``__default__`` entry. Assert at least one of these shapes + # holds. + assert ( + routing.get("__default__") == "replicate" + or routing.get("grains", "replicate") == "replicate" + ), f"Unspecified banks must default to 'replicate'; routing={routing!r}" + + # 2) Operator-supplied override must survive the merge, including + # an explicit override of the jobs default. + opts_override = {"cluster_cache_routing": {"jobs": "replicate", "grains": "ring"}} + routing2 = resolver(opts_override) # pylint: disable=not-callable + assert routing2.get("jobs") == "replicate", ( + "Operator override of jobs=replicate must win over the built-in " + f"'ring' default; got {routing2.get('jobs')!r}" + ) + assert routing2.get("grains") == "ring", ( + f"Operator-supplied grains=ring must survive the merge; got " + f"{routing2.get('grains')!r}" + ) + + +def test_cluster_cache_routing_defaults_include_none_entries(): + """ + Invariant: the merged routing dict MUST include the ``"none"`` + opt-outs for ``cluster/consensus`` and ``cluster/state_sync``. These + are the Raft-log and state-sync banks whose contents already + replicate via a dedicated transport; fanning them out through the + cache facade would overwrite each master's authoritative copy. + + Additionally, operator-supplied ``"none"`` entries must survive the + merge intact (not be coerced away or rewritten). + """ + import salt.config + + # 1) With no operator overrides, the defaults must inject "none" + # for the internal cluster banks. + routing = salt.config.get_cluster_cache_routing({}) + assert routing.get("cluster/consensus") == "none", ( + "Built-in default cluster/consensus=none must be present; got " + f"{routing.get('cluster/consensus')!r}" + ) + assert routing.get("cluster/state_sync") == "none", ( + "Built-in default cluster/state_sync=none must be present; got " + f"{routing.get('cluster/state_sync')!r}" + ) + + # 2) Operator-supplied "none" entry must survive the merge. + opts = {"cluster_cache_routing": {"custom_bank": "none"}} + routing = salt.config.get_cluster_cache_routing(opts) + assert routing.get("custom_bank") == "none", ( + f"Operator-supplied custom_bank=none must survive the merge; " + f"got {routing.get('custom_bank')!r}" + ) + # Built-in defaults must still be present alongside the operator entry. + assert routing.get("cluster/consensus") == "none" + assert routing.get("jobs") == "ring" + + +# --------------------------------------------------------------------------- +# Longest-prefix bank routing (compound-bank support + opt-out) +# --------------------------------------------------------------------------- + + +def test_cluster_consensus_banks_skipped(cluster_opts): + """ + Regression for cluster-bootstrap replication loop: Raft-persistence + writes to ``cluster/consensus//cluster`` and + ``cluster/consensus//cluster/log`` MUST NOT fan out through + the cache facade. If they did, peer masters would overwrite each + other's Raft logs on every heartbeat and the cluster would never + stabilise. + + This test exercises the longest-prefix match: the exact bank name + is not in ``_CLUSTER_CACHE_ROUTING_DEFAULTS``, only the + ``cluster/consensus`` prefix is. + """ + cache, _driver = _build_cache_with_mocked_driver(cluster_opts) + fanout = MagicMock(name="fanout") + delegate = MagicMock(name="delegate") + with _patch_cluster_fanout(fanout), _patch_delegate_fanout(delegate): + cache.store("cluster/consensus/nodeA/cluster", "term", {"term": 3}) + cache.store("cluster/consensus/nodeA/cluster/log", "17", {"index": 17}) + cache.store("cluster/state_sync/session-abc", "chunk-0", b"opaque") + + assert fanout.call_count == 0, ( + f"cluster/consensus and cluster/state_sync writes must NOT " + f"replicate; got {fanout.call_count} fanout events" + ) + assert delegate.call_count == 0, ( + f"cluster/consensus and cluster/state_sync writes must NOT " + f"ring-delegate; got {delegate.call_count} delegate events" + ) + + +def test_jobs_compound_bank_ring_routed(cluster_opts): + """ + Regression for the exact-match-vs-compound-bank latent bug: the + built-in ``{"jobs": "ring"}`` default used to miss compound banks + (``jobs/loads``, ``jobs/returns/``, ...) because the lookup + was ``routing.get(bank, "replicate")``. Longest-prefix match + fixes this: ``jobs/loads`` resolves to ``"ring"``. + """ + cache, _driver = _build_cache_with_mocked_driver(cluster_opts) + fanout = MagicMock(name="fanout") + delegate = MagicMock(name="delegate") + with _patch_cluster_fanout(fanout), _patch_delegate_fanout(delegate): + cache.store("jobs/loads", "jid123", {"jid": "jid123"}) + cache.store("jobs/returns/jid123", "minion-1", {"ret": "ok"}) + + # Two stores, both ring-routed, zero broadcast. + assert fanout.call_count == 0, ( + f"Compound jobs/* banks must ring-route (not broadcast); got " + f"{fanout.call_count} fanout events" + ) + assert delegate.call_count == 2, ( + f"Both jobs/* stores must fire the ring delegate; got " + f"{delegate.call_count} delegate events" + ) + + +def test_resolve_bank_routing_helper(): + """ + Direct unit coverage for the longest-prefix matcher so future + changes to the dispatch path don't silently regress the lookup. + """ + from salt.cache import _resolve_bank_routing + + routing = { + "jobs": "ring", + "cluster/consensus": "none", + "cluster/state_sync": "none", + } + # Exact match. + assert _resolve_bank_routing(routing, "jobs") == "ring" + # Single-segment prefix. + assert _resolve_bank_routing(routing, "jobs/loads") == "ring" + assert _resolve_bank_routing(routing, "jobs/returns/jid-9") == "ring" + # Multi-segment prefix. + assert _resolve_bank_routing(routing, "cluster/consensus/nodeA/cluster") == "none" + assert ( + _resolve_bank_routing(routing, "cluster/consensus/nodeA/cluster/log") == "none" + ) + assert _resolve_bank_routing(routing, "cluster/state_sync/session-1") == "none" + # No prefix match falls through to "replicate". + assert _resolve_bank_routing(routing, "grains") == "replicate" + assert _resolve_bank_routing(routing, "cluster/other/thing") == "replicate" + assert _resolve_bank_routing({}, "anything/at/all") == "replicate" diff --git a/tests/pytests/unit/cache/test_cluster_resource_register_replication.py b/tests/pytests/unit/cache/test_cluster_resource_register_replication.py new file mode 100644 index 000000000000..86543414ba76 --- /dev/null +++ b/tests/pytests/unit/cache/test_cluster_resource_register_replication.py @@ -0,0 +1,675 @@ +""" +Unit tests for cross-master resource-registry replication. + +Companion to ``test_cluster_cache_replication.py`` but targets the +``salt.utils.resource_registry.ResourceRegistry.register_minion`` +write path -- it bypasses ``salt.cache.Cache`` (hash-indexed mmap +with the SRN key shape) so the generic facade-level fan-out does +NOT cover it. These tests enforce that + +* ``AESFuncs.__register_resources_sync`` fires a replication event + after the local register lands. +* The peer-side handler applies the register locally via + ``update_resource_index`` with the same loop-guard shape + (``originating_master == self.opts["id"]`` -> drop) as the cache + fan-out. +* The event is gated on the same preconditions + (``cluster_id`` set, ``cluster_isolated_filesystem`` True, + ``cluster_cache_replication`` not forced False). +* Peer-side apply does NOT re-broadcast. +""" + +import pytest + +import salt.cache._cluster_replicator as replicator +from tests.support.mock import MagicMock, patch + + +@pytest.fixture +def cluster_opts(tmp_path): + """ + Cluster + isolated-FS opts -- the "replication should fire" shape. + """ + return { + "id": "master-1", + "cachedir": str(tmp_path), + "cluster_id": "c1", + "cluster_peers": ["master-2", "master-3"], + "cluster_isolated_filesystem": True, + "cluster_cache_replication": True, + "sock_dir": str(tmp_path), + } + + +@pytest.fixture +def shared_fs_opts(cluster_opts): + """Cluster but shared-FS -- replication must NOT fire.""" + cluster_opts["cluster_isolated_filesystem"] = False + return cluster_opts + + +@pytest.fixture +def replication_disabled_opts(cluster_opts): + """Cluster + isolated-FS but operator forced replication off.""" + cluster_opts["cluster_cache_replication"] = False + return cluster_opts + + +# --------------------------------------------------------------------------- +# Originator-side (fire) +# --------------------------------------------------------------------------- + + +def test_fire_resource_register_write_payload_shape(cluster_opts): + """ + Invariant: ``fire_resource_register_write`` emits an event whose + payload carries ``{minion_id, resources, originating_master}`` + with ``originating_master`` set to the local master id. + """ + captured = {} + + def fake_fire(opts, payload, tag=None): + captured["payload"] = payload + captured["tag"] = tag + + resources = {"bigip_vip": ["vip-1", "vip-2"]} + with patch.object(replicator, "_fire_payload", side_effect=fake_fire): + replicator.fire_resource_register_write(cluster_opts, "minion-a", resources) + + assert captured["payload"]["minion_id"] == "minion-a" + assert captured["payload"]["resources"] == resources + assert captured["payload"]["originating_master"] == cluster_opts["id"] + assert captured["tag"] == "cluster/runner/delegate_resource_register_write" + + +def test_register_resources_sync_fires_replication_event(cluster_opts, tmp_path): + """ + Invariant: in a clustered + isolated-FS deployment, + ``__register_resources_sync`` defers both the local write and the + peer fan-out to the publish daemon -- so the MWorker does NOT + call ``update_resource_index`` directly, and fires exactly one + replication event instead. This avoids double-writing peer-owned + SRNs when ring routing is active (the daemon has the ring state; + the MWorker doesn't). + """ + import salt.master + + aes = salt.master.AESFuncs.__new__(salt.master.AESFuncs) + aes.opts = cluster_opts + aes._AESFuncs__verify_load = lambda load, keys: load # noqa: SLF001 + load = {"id": "minion-a", "resources": {"bigip_vip": ["vip-1"]}} + + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) + ) as upd_mock, patch( + "salt.cache._cluster_replicator.fire_resource_register_write" + ) as fire_mock: + ret, fire_event, minion_id = aes._AESFuncs__register_resources_sync( + load + ) # noqa: SLF001 + + assert ret is True + assert upd_mock.call_count == 0, ( + "Clustered MWorker must defer local write to the publish daemon; " + f"got {upd_mock.call_count} direct writes" + ) + fire_mock.assert_called_once_with( + cluster_opts, + "minion-a", + load["resources"], + chunk_final=True, + chunk_scope="__all__", + chunk_index=None, + ) + + +def test_register_resources_sync_standalone_writes_locally(tmp_path): + """ + Invariant: a standalone (non-cluster) master writes to the mmap + registry directly inside the MWorker and never fires a + replication event. This is today's behavior for single-master + deployments and the defensive path when ``cluster_cache_replication`` + is forced off. + """ + import salt.master + + opts = { + "id": "standalone-1", + "cachedir": str(tmp_path), + "cluster_id": None, + "cluster_isolated_filesystem": False, + "sock_dir": str(tmp_path), + } + aes = salt.master.AESFuncs.__new__(salt.master.AESFuncs) + aes.opts = opts + aes._AESFuncs__verify_load = lambda load, keys: load # noqa: SLF001 + load = {"id": "minion-a", "resources": {"bigip_vip": ["vip-1"]}} + + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) + ) as upd_mock, patch( + "salt.cache._cluster_replicator.fire_resource_register_write" + ) as fire_mock: + aes._AESFuncs__register_resources_sync(load) # noqa: SLF001 + + upd_mock.assert_called_once_with( + opts, + "minion-a", + load["resources"], + True, + chunk_scope="__all__", + chunk_index=None, + ) + assert fire_mock.call_count == 0 + + +def test_register_resources_sync_no_fire_on_shared_fs(shared_fs_opts): + """ + Invariant: when ``cluster_isolated_filesystem`` is False the + peers already see the shared mmap; replication must NOT fire, + and the MWorker writes locally (no daemon deferral needed). + """ + import salt.master + + aes = salt.master.AESFuncs.__new__(salt.master.AESFuncs) + aes.opts = shared_fs_opts + aes._AESFuncs__verify_load = lambda load, keys: load # noqa: SLF001 + load = {"id": "minion-a", "resources": {"bigip_vip": ["vip-1"]}} + + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) + ) as upd_mock, patch( + "salt.cache._cluster_replicator.fire_resource_register_write" + ) as fire_mock: + aes._AESFuncs__register_resources_sync(load) # noqa: SLF001 + + upd_mock.assert_called_once_with( + shared_fs_opts, + "minion-a", + load["resources"], + True, + chunk_scope="__all__", + chunk_index=None, + ) + assert fire_mock.call_count == 0, ( + "Shared-FS cluster must not fan-out resource-register writes; " + f"got {fire_mock.call_count} events" + ) + + +def test_register_resources_sync_no_fire_when_replication_disabled( + replication_disabled_opts, +): + """ + Invariant: ``cluster_cache_replication: False`` forces the + replication off even in a cluster + isolated-FS topology. + """ + import salt.master + + aes = salt.master.AESFuncs.__new__(salt.master.AESFuncs) + aes.opts = replication_disabled_opts + aes._AESFuncs__verify_load = lambda load, keys: load # noqa: SLF001 + load = {"id": "minion-a", "resources": {"bigip_vip": ["vip-1"]}} + + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) + ), patch( + "salt.cache._cluster_replicator.fire_resource_register_write" + ) as fire_mock: + aes._AESFuncs__register_resources_sync(load) # noqa: SLF001 + + assert fire_mock.call_count == 0 + + +def test_register_resources_sync_no_fire_on_standalone(tmp_path): + """ + Invariant: a standalone (non-cluster) master must not fire the + replication event -- there are no peers to send it to. + """ + import salt.master + + opts = { + "id": "standalone-1", + "cachedir": str(tmp_path), + "cluster_id": None, + "cluster_isolated_filesystem": False, + "sock_dir": str(tmp_path), + } + aes = salt.master.AESFuncs.__new__(salt.master.AESFuncs) + aes.opts = opts + aes._AESFuncs__verify_load = lambda load, keys: load # noqa: SLF001 + load = {"id": "minion-a", "resources": {"bigip_vip": ["vip-1"]}} + + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) + ), patch( + "salt.cache._cluster_replicator.fire_resource_register_write" + ) as fire_mock: + aes._AESFuncs__register_resources_sync(load) # noqa: SLF001 + + assert fire_mock.call_count == 0 + + +# --------------------------------------------------------------------------- +# Peer-side (handle) +# --------------------------------------------------------------------------- + + +def test_handle_delegate_resource_register_write_loop_guard(cluster_opts): + """ + Invariant: a peer master receiving a resource-register event whose + ``originating_master`` equals its own ``opts["id"]`` MUST drop + the write. Prevents bus echo from re-applying the register (and + re-firing the fan-out). + """ + payload = { + "minion_id": "minion-a", + "resources": {"bigip_vip": ["vip-1"]}, + "originating_master": cluster_opts["id"], # == self + } + upd_mock = MagicMock() + with patch("salt.utils.minions.update_resource_index_chunk", upd_mock): + result = replicator._handle_delegate_resource_register_write( + cluster_opts, payload + ) + assert result is False + assert upd_mock.call_count == 0, ( + "Loop-guard must drop self-originated event; " + f"update_resource_index was called {upd_mock.call_count} times" + ) + + +def test_handle_delegate_resource_register_write_applies_locally(cluster_opts): + """ + Invariant: a peer master receiving a foreign-origin register must + call ``update_resource_index(opts, minion_id, resources)`` exactly + once. + """ + payload = { + "minion_id": "minion-a", + "resources": {"bigip_vip": ["vip-1", "vip-2"]}, + "originating_master": "master-2", + } + upd_mock = MagicMock(return_value=(2, 0)) + with patch("salt.utils.minions.update_resource_index_chunk", upd_mock): + result = replicator._handle_delegate_resource_register_write( + cluster_opts, payload + ) + assert result is True + upd_mock.assert_called_once_with( + cluster_opts, + "minion-a", + payload["resources"], + True, + chunk_scope="__all__", + chunk_index=None, + ) + + +def test_handle_delegate_resource_register_write_missing_fields(cluster_opts): + """ + Invariant: malformed payloads (missing ``minion_id`` or + ``resources``) are dropped rather than crashing the receiver. + """ + upd_mock = MagicMock() + with patch("salt.utils.minions.update_resource_index_chunk", upd_mock): + # Missing minion_id + assert ( + replicator._handle_delegate_resource_register_write( + cluster_opts, + {"resources": {}, "originating_master": "master-2"}, + ) + is False + ) + # Missing resources + assert ( + replicator._handle_delegate_resource_register_write( + cluster_opts, + {"minion_id": "x", "originating_master": "master-2"}, + ) + is False + ) + assert upd_mock.call_count == 0 + + +# --------------------------------------------------------------------------- +# Ring-routed dispatch (1-of-N via ring_membership) -- publish-daemon side +# --------------------------------------------------------------------------- + + +@pytest.fixture +def ring_cluster_opts(cluster_opts): + """ + Three-master cluster opts matching the ring voter set used in the + ring-routing tests. + """ + cluster_opts["id"] = "master-1" + cluster_opts["cluster_peers"] = ["master-2", "master-3"] + return cluster_opts + + +@pytest.fixture +def populated_ring(): + """ + Rebuild the ``resource_index`` ring with three voters for the + duration of a test, then reset the registry. Three voters gives + deterministic 1-of-3 ownership for any SRN. + """ + import salt.cluster.ring_membership as rm + import salt.utils.resource_registry as rr + + rm.reset() + rm.rebuild(rr.RESOURCE_INDEX_RING_NAME, ["master-1", "master-2", "master-3"]) + try: + yield rm.get_ring(rr.RESOURCE_INDEX_RING_NAME) + finally: + rm.reset() + + +def test_partition_for_dispatch_ring_active(ring_cluster_opts, populated_ring): + """ + Invariant: with the ``resource_index`` ring populated, + :func:`partition_resources_for_dispatch` returns a local bucket + containing only self-owned SRNs, peer buckets keyed by remote + owner, and ``ring_active=True``. + """ + import salt.cluster + import salt.utils.resource_registry as rr + + resources = {"dummy": [f"dummy-{i:04d}" for i in range(30)]} + local, peers, ring_active = replicator.partition_resources_for_dispatch( + ring_cluster_opts, resources + ) + + assert ring_active is True + self_node = salt.cluster.node_id(ring_cluster_opts) + + # Build the expected ownership map. + expected = {} + for rid in resources["dummy"]: + srn = rr.resource_index_srn_key("dummy", rid) + owner = populated_ring.get_owner(srn) + expected.setdefault(owner, []).append(rid) + + # Local bucket == self_node's bucket from expected. + local_rids = sorted(local.get("dummy", [])) + assert local_rids == sorted(expected.get(self_node, [])), ( + f"Local partition mismatch; expected {sorted(expected.get(self_node, []))}, " + f"got {local_rids}" + ) + + # Peer buckets == remaining owners in expected. + peer_rids_by_owner = { + owner: sorted(r.get("dummy", [])) for owner, r in peers.items() + } + expected_peer_rids = { + owner: sorted(rids) for owner, rids in expected.items() if owner != self_node + } + assert peer_rids_by_owner == expected_peer_rids + + +def test_partition_for_dispatch_partition_disjoint(ring_cluster_opts, populated_ring): + """ + Invariant (partition): for any SRN, exactly one bucket owns it. + No SRN appears in both ``local`` and any peer bucket, and no SRN + appears in more than one peer bucket. + """ + resources = {"dummy": [f"dummy-{i:04d}" for i in range(50)]} + local, peers, _ = replicator.partition_resources_for_dispatch( + ring_cluster_opts, resources + ) + + seen = set() + for rid in local.get("dummy", []): + assert rid not in seen, f"Duplicate SRN in local bucket: {rid}" + seen.add(rid) + for owner, owner_resources in peers.items(): + for rid in owner_resources.get("dummy", []): + assert ( + rid not in seen + ), f"SRN {rid} appeared in a prior bucket and in peer {owner}" + seen.add(rid) + # Every input SRN landed in exactly one bucket. + assert seen == set(resources["dummy"]) + + +def test_partition_for_dispatch_ring_empty_returns_broadcast(ring_cluster_opts): + """ + Invariant: empty/absent ring returns the full resources as the + local bucket, no peer buckets, and ``ring_active=False`` so the + caller falls through to today's broadcast semantics. + """ + import salt.cluster.ring_membership as rm + + rm.reset() # Empty ring -- no voters + resources = {"dummy": ["a", "b", "c"]} + local, peers, ring_active = replicator.partition_resources_for_dispatch( + ring_cluster_opts, resources + ) + assert ring_active is False + assert local == resources + assert peers == {} + + +def test_fire_resource_register_write_always_emits_single_event(ring_cluster_opts): + """ + Invariant: ``fire_resource_register_write`` is a thin MWorker-side + emit that always publishes one event with the full resources + dict. The publish daemon's dispatcher owns the ring vs + broadcast decision -- not this function. + """ + import salt.cluster.ring_membership as rm + + rm.reset() + emitted = [] + with patch.object( + replicator, + "_fire_payload", + side_effect=lambda o, p, tag=None: emitted.append((p, tag)), + ): + resources = {"dummy": ["a", "b", "c"]} + replicator.fire_resource_register_write( + ring_cluster_opts, "minion-a", resources + ) + assert len(emitted) == 1 + payload, tag = emitted[0] + assert tag == "cluster/runner/delegate_resource_register_write" + assert payload["resources"] == resources + assert payload["minion_id"] == "minion-a" + assert payload["originating_master"] == ring_cluster_opts["id"] + assert "target_master" not in payload + + +def test_register_resources_sync_cluster_mode_defers_to_daemon( + ring_cluster_opts, +): + """ + Invariant: in a clustered + isolated-FS deployment the MWorker + does NOT call ``update_resource_index`` directly -- it fires a + replication event and lets the publish daemon apply the local + write under ring-aware partitioning. This is the structural + guard against the MWorker's fork-time-empty ring state causing a + double-write of peer-owned SRNs. + """ + import salt.master + + aes = salt.master.AESFuncs.__new__(salt.master.AESFuncs) + aes.opts = ring_cluster_opts + aes._AESFuncs__verify_load = lambda load, keys: load # noqa: SLF001 + resources = {"dummy": [f"dummy-{i:04d}" for i in range(10)]} + load = {"id": "minion-a", "resources": resources} + + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(0, 0) + ) as upd_mock, patch( + "salt.cache._cluster_replicator.fire_resource_register_write" + ) as fire_mock: + ret, _, _ = aes._AESFuncs__register_resources_sync(load) # noqa: SLF001 + + assert ret is True + assert ( + upd_mock.call_count == 0 + ), "Clustered MWorker must defer the local write to the publish daemon" + fire_mock.assert_called_once_with( + ring_cluster_opts, + "minion-a", + resources, + chunk_final=True, + chunk_scope="__all__", + chunk_index=None, + ) + + +def test_handle_delegate_resource_register_write_propagates_chunk_index( + cluster_opts, +): + """ + Invariant: the peer-side handler must forward ``chunk_index`` from + the payload into ``update_resource_index_chunk`` so the + registry's out-of-order gap check can detect missing earlier chunks. + Without this plumbing, the registry would see ``chunk_index=None`` + on every arrival and fall back to legacy "prune-every-final" + semantics -- the pre-fix behavior. + """ + payload = { + "minion_id": "minion-a", + "resources": {"dummy": ["d0"]}, + "originating_master": "master-2", + "chunk_final": True, + "chunk_scope": "dummy", + "chunk_index": 3, + } + upd_mock = MagicMock(return_value=(1, 0)) + with patch("salt.utils.minions.update_resource_index_chunk", upd_mock): + replicator._handle_delegate_resource_register_write(cluster_opts, payload) + upd_mock.assert_called_once_with( + cluster_opts, + "minion-a", + {"dummy": ["d0"]}, + True, + chunk_scope="dummy", + chunk_index=3, + ) + + +def test_handle_delegate_resource_register_write_out_of_order_no_srn_loss( + cluster_opts, tmp_path +): + """ + End-to-end peer-side test for the Phase-3 reviewer finding #14 + (chunk-reordering race). Feed four chunks for the same minion + + scope in reverse order to the peer-side handler through multiple + threads; the final result must retain every SRN. + + Pre-fix (no lock / no gap check): + - The terminal chunk (chunk_index=3, chunk_final=True) could race + ahead of earlier chunks, run the prune pass against a + 1-SRN-wide pending window, and wipe the registry. + - Concurrent ``_pending_registers`` mutation could drop SRNs via + lost-update on the plain dict. + + Post-fix: + - The publish-daemon dispatch routes every event for a minion + through a dedicated single-worker executor + (``_get_peer_apply_executor``) so chunk arrivals serialize + in-submission-order per minion. This test bypasses the + publish daemon and calls the free-function handler directly, + so the executor FIFO does NOT serialize the three threads -- + what rescues ordering here is the inner gap-check on + ``_pending_registers``. + - The ``register_minion_chunk`` gap check defers the terminal + prune until every ``chunk_index`` in ``0..final_index`` has + arrived, closing the ordering hole the executor lacks. + """ + import salt.utils.resource_registry + + salt.utils.resource_registry.reset_registry() + opts = dict(cluster_opts) + opts["cachedir"] = str(tmp_path) + opts["resource_index_primary_capacity"] = 4096 + opts["resource_index_primary_slot_size"] = 128 + opts["id"] = "master-1" + + def payload_for(idx, rid, is_final): + return { + "minion_id": "mx", + "resources": {"dummy": [rid]}, + "originating_master": "master-2", + "chunk_final": is_final, + "chunk_scope": "dummy", + "chunk_index": idx, + } + + # Deliver the final chunk FIRST (worst-case race ordering), then + # chunks 2, 1, 0 in that order via separate threads. + all_payloads = [ + payload_for(3, "d3", True), + payload_for(2, "d2", False), + payload_for(1, "d1", False), + payload_for(0, "d0", False), + ] + + import threading + + errs = [] + err_lock = threading.Lock() + + def deliver(p): + try: + replicator._handle_delegate_resource_register_write(opts, p) + except Exception as exc: # pylint: disable=broad-except + with err_lock: + errs.append(exc) + + # The first delivery (the final chunk) is single-threaded to make + # the ordering deterministic for the pre-fix regression. The + # remaining three go in parallel to exercise the per-scope apply + # lock. + deliver(all_payloads[0]) + threads = [threading.Thread(target=deliver, args=(p,)) for p in all_payloads[1:]] + for t in threads: + t.start() + for t in threads: + t.join() + assert not errs, f"peer-side apply raised: {errs}" + + registry = salt.utils.resource_registry.get_registry(opts) + try: + resources = registry.get_resources_for_minion("mx") + got = sorted(resources.get("dummy", [])) + assert got == ["d0", "d1", "d2", "d3"], ( + "Out-of-order peer-side chunk delivery dropped SRNs; " + f"expected [d0..d3], got {got}" + ) + finally: + salt.utils.resource_registry.reset_registry() + + +def test_handle_delegate_resource_register_write_no_re_broadcast(cluster_opts): + """ + Invariant: applying a peer-originated register must NOT re-trigger + a secondary fan-out. ``update_resource_index`` is a direct call + to the mmap registry (no cache facade), so there is no + ``_already_replicated``-equivalent flag to thread through -- the + non-broadcasting property is structural: the handler calls + ``update_resource_index`` and nothing else, and + ``update_resource_index`` itself has no replication seam. + + Patch the originator-side seam and assert it's never touched when + the peer-side handler runs. + """ + payload = { + "minion_id": "minion-a", + "resources": {"bigip_vip": ["vip-1"]}, + "originating_master": "master-2", + } + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) + ), patch( + "salt.cache._cluster_replicator.fire_resource_register_write" + ) as fire_mock: + replicator._handle_delegate_resource_register_write(cluster_opts, payload) + assert fire_mock.call_count == 0, ( + "Peer-side apply must not re-fire the replication event; " + f"got {fire_mock.call_count} secondary emits" + ) diff --git a/tests/pytests/unit/channel/test_cluster_discover_retry.py b/tests/pytests/unit/channel/test_cluster_discover_retry.py new file mode 100644 index 000000000000..d9827ca70e5f --- /dev/null +++ b/tests/pytests/unit/channel/test_cluster_discover_retry.py @@ -0,0 +1,176 @@ +""" +Unit tests for the periodic ``cluster/peer/discover`` retry scheduled by +``MasterPubServerChannel._schedule_discover_retry``. + +The initial ``discover_peers`` call in ``salt.master.Master.start`` is +fire-and-forget: if the peer's pool-puller TCP listener is not yet bound +when the discover event is forwarded, the push raises ``StreamClosedError`` +and the discover is lost. Without a retry the joining master waits +forever on ``_discover_event`` and the cluster bootstrap stalls. These +tests lock in that the retry: + +* re-sends discover on the configured interval when no peer has replied, +* stops immediately once every configured peer has registered its AES key, +* stops after ``cluster_join_discover_retry_timeout`` seconds have elapsed. +""" + +import tornado.ioloop + +import salt.channel.server +from tests.support.mock import MagicMock, patch + + +def _opts(**overrides): + opts = { + "cluster_id": "test-cluster", + "cluster_peers": ["peer-a", "peer-b"], + "cluster_pool_port": 4520, + "sock_dir": "/tmp/does-not-matter", + "interface": "127.0.0.1", + "publish_port": 4505, + "cluster_join_discover_retry_interval": 2, + "cluster_join_discover_retry_timeout": 60, + } + opts.update(overrides) + return opts + + +class _FakeLoop: + """Minimal io_loop stand-in that exposes ``time()``.""" + + def __init__(self, start=0.0): + self._t = start + + def time(self): + return self._t + + def advance(self, seconds): + self._t += seconds + + +def _make_channel(opts): + """Instantiate the channel without touching disk or crypto.""" + chan = salt.channel.server.MasterPubServerChannel.__new__( + salt.channel.server.MasterPubServerChannel + ) + chan.opts = opts + chan.cluster_peers = list(opts["cluster_peers"]) + chan.peer_keys = {} + chan._discover_event = None + chan._discover_token = None + chan._discover_candidates = {} + chan._discover_replies = set() + chan._discover_retry_pc = None + chan.io_loop = _FakeLoop() + return chan + + +def test_pending_discover_peers_excludes_peers_that_replied(): + chan = _make_channel(_opts(cluster_peers=["peer-a", "peer-b", "peer-c"])) + chan._discover_replies = {"peer-a"} + pending = chan._pending_discover_peers() + assert "peer-a" not in pending + assert set(pending) == {"peer-b", "peer-c"} + + +def test_pending_discover_peers_is_not_cancelled_by_aes_key_arrival(): + """ + AES-key broadcast populates ``peer_keys`` independently of the + discover protocol. A peer that has merely sent us its AES key has + *not* answered discover, and the retry must keep firing. + """ + chan = _make_channel(_opts(cluster_peers=["peer-a", "peer-b"])) + chan.peer_keys = {"peer-a": b"key-a", "peer-b": b"key-b"} + chan._discover_replies = set() + assert chan._pending_discover_peers() == ["peer-a", "peer-b"] + + +def test_pending_discover_peers_strips_port_suffix_when_matching(): + chan = _make_channel(_opts(cluster_peers=["peer-a:4520", "peer-b"])) + # Discover-reply handler keys ``_discover_replies`` by the bare host + # (``payload["peer_id"]``); match that against the ``host:port`` form + # an operator may have used in ``cluster_peers``. + chan._discover_replies = {"peer-a"} + assert chan._pending_discover_peers() == ["peer-b"] + + +def test_schedule_discover_retry_resends_until_reply(): + chan = _make_channel(_opts()) + captured = [] + + def _fake_discover(peers=None): + # Grab a snapshot so later retries don't mutate the recorded list. + captured.append(list(peers) if peers is not None else None) + + chan.discover_peers = _fake_discover + + pc = MagicMock() + with patch.object(tornado.ioloop, "PeriodicCallback", return_value=pc) as pc_cls: + chan._schedule_discover_retry() + + # PeriodicCallback was constructed with the configured interval. + pc_cls.assert_called_once() + callback = pc_cls.call_args.args[0] + assert pc_cls.call_args.args[1] == 2000 # 2 seconds in ms + pc.start.assert_called_once() + assert chan._discover_retry_pc is pc + + # Fire two retry ticks with no peer having replied. + callback() + callback() + + assert captured == [["peer-a", "peer-b"], ["peer-a", "peer-b"]] + pc.stop.assert_not_called() + + # Peer A replies; the next tick should only retry peer B. + chan._discover_replies.add("peer-a") + callback() + assert captured[-1] == ["peer-b"] + pc.stop.assert_not_called() + + # Peer B replies; the next tick must cancel the retry. + chan._discover_replies.add("peer-b") + callback() + pc.stop.assert_called_once() + assert chan._discover_retry_pc is None + + +def test_schedule_discover_retry_stops_on_timeout(): + chan = _make_channel( + _opts( + cluster_join_discover_retry_interval=1, + cluster_join_discover_retry_timeout=10, + ) + ) + chan.discover_peers = MagicMock() + pc = MagicMock() + with patch.object(tornado.ioloop, "PeriodicCallback", return_value=pc) as pc_cls: + chan._schedule_discover_retry() + callback = pc_cls.call_args.args[0] + + # No peer replies; advance beyond the timeout and tick. + chan.io_loop.advance(11) + callback() + + # Discover is NOT re-sent after the deadline; the retry is cancelled. + assert chan.discover_peers.call_count == 0 + pc.stop.assert_called() + assert chan._discover_retry_pc is None + + +def test_schedule_discover_retry_no_peers_is_noop(): + chan = _make_channel(_opts(cluster_peers=[])) + with patch.object(tornado.ioloop, "PeriodicCallback") as pc_cls: + chan._schedule_discover_retry() + pc_cls.assert_not_called() + assert chan._discover_retry_pc is None + + +def test_schedule_discover_retry_cancels_prior_instance(): + chan = _make_channel(_opts()) + prior = MagicMock() + chan._discover_retry_pc = prior + with patch.object(tornado.ioloop, "PeriodicCallback") as pc_cls: + chan._schedule_discover_retry() + prior.stop.assert_called_once() + pc_cls.assert_called_once() diff --git a/tests/pytests/unit/channel/test_cluster_state_sync_cache_banks.py b/tests/pytests/unit/channel/test_cluster_state_sync_cache_banks.py new file mode 100644 index 000000000000..80d470ccade3 --- /dev/null +++ b/tests/pytests/unit/channel/test_cluster_state_sync_cache_banks.py @@ -0,0 +1,238 @@ +""" +Unit tests for the join-time catch-up of replicate-mode cache banks. + +Covers four behaviours: + +* ``enumerate_replicate_banks`` walks the cachedir and filters out + ``jobs`` / ``cluster/consensus`` / ``cluster/state_sync``. +* ``iter_cache_bank_chunks`` respects the per-chunk byte budget. +* ``install_bank_sync_chunk`` writes with ``_already_replicated=True`` + so the receiver does not re-broadcast. +* The barrier methods on ``MasterPubServerChannel`` queue a live + delegate-cache-write until the matching ``bank_sync:`` + channel eofs and then replay the queue. +""" + +import salt.cache +import salt.cluster.state_sync as ss + +# --------------------------------------------------------------------------- +# enumerate_replicate_banks +# --------------------------------------------------------------------------- + + +def test_enumerate_replicate_banks_skips_internal_banks(tmp_path, monkeypatch): + """ + ``jobs``, ``cluster/consensus`` and ``cluster/state_sync`` must not + appear in the catch-up list even if their directories exist in the + cachedir. + """ + cachedir = tmp_path / "cache" + cachedir.mkdir() + for name in ("grains", "resources", "jobs", "minions"): + (cachedir / name).mkdir() + # The ``cluster`` subtree is skipped wholesale. + (cachedir / "cluster").mkdir() + (cachedir / "cluster" / "consensus").mkdir() + (cachedir / "cluster" / "state_sync").mkdir() + opts = {"cachedir": str(cachedir), "cluster_cache_routing": {}} + banks = ss.enumerate_replicate_banks(opts) + assert "jobs" not in banks + assert "cluster" not in banks + # ``grains`` / ``resources`` / ``minions`` all fall through the + # longest-prefix matcher as "replicate" by default. + assert "grains" in banks + assert "resources" in banks + assert "minions" in banks + + +def test_enumerate_replicate_banks_honours_operator_routing(tmp_path): + """ + Operator-set ``cluster_cache_routing`` entries must suppress + replicate for named banks. + """ + cachedir = tmp_path / "cache" + cachedir.mkdir() + for name in ("grains", "secret_bank"): + (cachedir / name).mkdir() + opts = { + "cachedir": str(cachedir), + "cluster_cache_routing": {"secret_bank": "none"}, + } + banks = ss.enumerate_replicate_banks(opts) + assert "grains" in banks + assert "secret_bank" not in banks + + +def test_enumerate_replicate_banks_empty_cachedir(tmp_path): + opts = {"cachedir": str(tmp_path / "does_not_exist")} + assert ss.enumerate_replicate_banks(opts) == [] + + +# --------------------------------------------------------------------------- +# iter_cache_bank_chunks (byte-budget chunking) +# --------------------------------------------------------------------------- + + +class _FakeBankCache: + """Minimal ``salt.cache.Cache``-compatible stub for chunker tests.""" + + def __init__(self, dump): + self._dump = dump + + def list_all(self, bank, include_data=True): + return self._dump + + def store(self, bank, key, value, expires=None, _already_replicated=False): + self.last_store = (bank, key, value, _already_replicated) + + +def test_iter_cache_bank_chunks_respects_byte_budget(monkeypatch): + """ + With a small ``byte_budget`` every chunk should carry at most one + big item. + """ + big = "x" * 4096 + dump = {f"key-{i}": big for i in range(10)} + fake = _FakeBankCache(dump) + monkeypatch.setattr(salt.cache, "Cache", lambda opts, driver=None: fake) + chunks = list( + ss.iter_cache_bank_chunks({"cachedir": "/tmp"}, "grains", byte_budget=512) + ) + # Every chunk holds >=1 entry; many chunks overall. + assert all(len(chunk) >= 1 for chunk in chunks) + assert len(chunks) >= 2 + # Every original key must appear exactly once across all chunks. + seen = [item["key"] for chunk in chunks for item in chunk] + assert sorted(seen) == sorted(dump.keys()) + + +def test_iter_cache_bank_chunks_empty_bank_yields_one_empty(monkeypatch): + fake = _FakeBankCache({}) + monkeypatch.setattr(salt.cache, "Cache", lambda opts, driver=None: fake) + chunks = list(ss.iter_cache_bank_chunks({"cachedir": "/tmp"}, "grains")) + assert chunks == [[]] + + +# --------------------------------------------------------------------------- +# install_bank_sync_chunk sets _already_replicated=True +# --------------------------------------------------------------------------- + + +def test_install_bank_sync_chunk_sets_already_replicated(monkeypatch): + """ + The join-time install path must not re-broadcast -- every + ``cache.store`` call carries ``_already_replicated=True``. + """ + fake = _FakeBankCache({}) + monkeypatch.setattr(salt.cache, "Cache", lambda opts, driver=None: fake) + items = [{"key": "mid-a", "value": {"grain": "x"}}] + written = ss.install_bank_sync_chunk({"cachedir": "/tmp"}, "grains", items) + assert written == 1 + assert fake.last_store == ("grains", "mid-a", {"grain": "x"}, True) + + +# --------------------------------------------------------------------------- +# Barrier: queue-then-replay on MasterPubServerChannel +# --------------------------------------------------------------------------- + + +class _BarrierHarness: + """ + Minimal harness that exposes just the barrier helpers from + ``MasterPubServerChannel`` so they can be exercised without the + full publish-daemon machinery. Mirrors the attributes those + helpers actually touch. + """ + + def __init__(self): + self.opts = {"id": "master-A"} + self._cache_bank_barriers = {} + self.applied = [] + + # Pull the barrier methods off the real class so this harness + # drifts in lockstep if the implementation changes. + from salt.channel.server import MasterPubServerChannel + + _cache_bank_barrier_active = MasterPubServerChannel._cache_bank_barrier_active + _queue_cache_bank_delta = MasterPubServerChannel._queue_cache_bank_delta + _release_cache_bank_barrier = MasterPubServerChannel._release_cache_bank_barrier + _drain_cache_bank_barrier = MasterPubServerChannel._drain_cache_bank_barrier + + +def test_barrier_queues_live_delta_until_eof(monkeypatch): + """ + A live ``delegate-cache-write`` for a bank that still has a + pending catch-up stream is queued, then applied on the matching + ``bank_sync:`` eof. + """ + harness = _BarrierHarness() + harness._cache_bank_barriers = {"S": {"grains": []}} + applied = [] + + def fake_apply(opts, payload): + applied.append(payload) + return True + + monkeypatch.setattr( + "salt.cache._cluster_replicator._handle_delegate_cache_write", + fake_apply, + ) + + # Bank matches -> queued, not applied. + payload1 = {"bank": "grains", "key": "mid-a", "data": {"grain": "x"}} + assert harness._cache_bank_barrier_active("grains") is True + harness._queue_cache_bank_delta("grains", payload1) + assert harness._cache_bank_barriers["S"]["grains"] == [payload1] + assert applied == [] + + # Compound bank (grains/mid-b) matches longest-prefix on "grains". + payload2 = {"bank": "grains/mid-b", "key": "mid-b", "data": {"g": "y"}} + assert harness._cache_bank_barrier_active("grains/mid-b") is True + harness._queue_cache_bank_delta("grains/mid-b", payload2) + assert len(harness._cache_bank_barriers["S"]["grains"]) == 2 + + # eof arrives -> both queued payloads are replayed in order. + harness._release_cache_bank_barrier("S", "grains") + assert applied == [payload1, payload2] + # Barrier for this session is gone. + assert "S" not in harness._cache_bank_barriers + + +def test_barrier_inactive_outside_pending_set(): + """ + A bank name not in any session's barrier set reports + ``_cache_bank_barrier_active`` False (so the normal apply path + is used). + """ + harness = _BarrierHarness() + harness._cache_bank_barriers = {"S": {"grains": []}} + assert harness._cache_bank_barrier_active("resources") is False + + +def test_barrier_drain_fires_everything(monkeypatch): + """ + ``_drain_cache_bank_barrier`` must flush all pending banks for a + session (used by on_complete when the deadline forces completion). + """ + harness = _BarrierHarness() + applied = [] + + def fake_apply(opts, payload): + applied.append(payload) + return True + + monkeypatch.setattr( + "salt.cache._cluster_replicator._handle_delegate_cache_write", + fake_apply, + ) + + harness._cache_bank_barriers = { + "S": { + "grains": [{"bank": "grains", "key": "a"}], + "resources": [{"bank": "resources", "key": "b"}], + } + } + harness._drain_cache_bank_barrier("S") + assert sorted(p["bank"] for p in applied) == ["grains", "resources"] + assert "S" not in harness._cache_bank_barriers diff --git a/tests/pytests/unit/cli/test_caller_resources.py b/tests/pytests/unit/cli/test_caller_resources.py index 1ddcf78e2336..6cbaed7a2225 100644 --- a/tests/pytests/unit/cli/test_caller_resources.py +++ b/tests/pytests/unit/cli/test_caller_resources.py @@ -3,9 +3,11 @@ (``salt.cli.caller.BaseCaller._call_with_resources``). The dispatch path runs entirely in-process against an ``SMinion`` whose -``resources`` config is populated from Pillar. These tests use the dummy -resource type (``salt.resources.dummy``) so they exercise the real loader -and matcher code without needing an actual master/minion daemon. +resource inventory is declared via the ``resource_types`` minion opt and +each type's own per-module opts (here, ``dummy_resource_count``). These +tests use the dummy resource type (``salt.resources.dummy``) so they +exercise the real loader and matcher code without needing an actual +master/minion daemon. Coverage: @@ -57,9 +59,13 @@ def call_opts(tmp_path): opts["file_client"] = "local" opts["file_roots"] = {"base": [str(tmp_path / "srv" / "salt")]} opts["pillar_roots"] = {"base": [str(tmp_path / "srv" / "pillar")]} - opts["pillar"] = { - "resources": {"dummy": {"resource_ids": ["dummy-01", "dummy-02", "dummy-03"]}} - } + # Opts-driven resource inventory: ``resource_types`` enables the dummy + # loader, and ``dummy_resource_count`` controls how many synthetic IDs + # ``salt.resources.dummy.discover(opts)`` yields. The IDs follow the + # module's fixed ``dummy-NNNNNNN`` (7-digit zero-padded) pattern. + opts["resource_types"] = ["dummy"] + opts["dummy_resource_count"] = 3 + opts.setdefault("pillar", {}) opts.setdefault("grains", {}) # Caller defaults — all targeting flags off / at their parser defaults. opts["fun"] = "test.ping" @@ -106,36 +112,36 @@ def _shouldnt_run(self): def test_r_default_star_includes_minion_and_all_resources(call_opts): - """salt-call -r test.ping → {minion: True, dummy-01: True, ...}.""" + """salt-call -r test.ping → {minion: True, dummy-0000001: True, ...}.""" call_opts["resources_dispatch"] = True caller = _build_caller(call_opts) ret = caller._call_with_resources() payload = ret["return"] assert isinstance(payload, dict), payload assert payload[call_opts["id"]] is True - for rid in ("dummy-01", "dummy-02", "dummy-03"): + for rid in ("dummy-0000001", "dummy-0000002", "dummy-0000003"): assert payload[rid] is True, (rid, payload) assert ret["retcode"] == salt.defaults.exitcodes.EX_OK def test_r_bare_id_glob_returns_single_value(call_opts): - """salt-call -r --tgt dummy-02 test.ping → True (single, unwrapped).""" + """salt-call -r --tgt dummy-0000002 test.ping → True (single, unwrapped).""" call_opts["resources_dispatch"] = True - call_opts["resources_tgt"] = "dummy-02" + call_opts["resources_tgt"] = "dummy-0000002" caller = _build_caller(call_opts) ret = caller._call_with_resources() assert ret["return"] is True def test_r_list_targets_returns_dict_of_those_only(call_opts): - """-r --tgt 'dummy-01,dummy-03' --tgt-type list → only those resources.""" + """-r --tgt 'dummy-0000001,dummy-0000003' --tgt-type list → only those.""" call_opts["resources_dispatch"] = True - call_opts["resources_tgt"] = "dummy-01,dummy-03" + call_opts["resources_tgt"] = "dummy-0000001,dummy-0000003" call_opts["resources_tgt_type"] = "list" caller = _build_caller(call_opts) payload = caller._call_with_resources()["return"] assert isinstance(payload, dict) - assert set(payload.keys()) == {"dummy-01", "dummy-03"} + assert set(payload.keys()) == {"dummy-0000001", "dummy-0000003"} assert all(v is True for v in payload.values()) @@ -148,24 +154,24 @@ def test_r_pure_compound_excludes_managing_minion(call_opts): payload = caller._call_with_resources()["return"] assert isinstance(payload, dict) assert call_opts["id"] not in payload - assert set(payload.keys()) == {"dummy-01", "dummy-02", "dummy-03"} + assert set(payload.keys()) == {"dummy-0000001", "dummy-0000002", "dummy-0000003"} def test_r_grains_items_not_supported_without_override(call_opts): """ - ``salt-call -r --tgt dummy-01 grains.items`` — the ``dummy`` resource - type ships no ``grains.py`` override, so ``grains.items`` is not - reachable via the per-resource loader (deny-by-default surface, - #69881 fix). The caller returns the "not supported for resource - type" rejection instead of falling through to the managing minion's - stock ``grains.items``. + ``salt-call -r --tgt dummy-0000001 grains.items`` — the ``dummy`` + resource type ships no ``grains.py`` override, so ``grains.items`` + is not reachable via the per-resource loader (deny-by-default + surface, #69881 fix). The caller returns the "not supported for + resource type" rejection instead of falling through to the managing + minion's stock ``grains.items``. A type that legitimately wants to expose grains still can — by shipping ``resources//modules/grains.py`` that returns the resource's grains (or thin-wraps ``__minion__["grains.items"]``). """ call_opts["resources_dispatch"] = True - call_opts["resources_tgt"] = "dummy-01" + call_opts["resources_tgt"] = "dummy-0000001" call_opts["fun"] = "grains.items" caller = _build_caller(call_opts) payload = caller._call_with_resources()["return"] @@ -187,7 +193,7 @@ def test_r_grains_items_not_supported_per_target(call_opts): caller = _build_caller(call_opts) payload = caller._call_with_resources()["return"] assert isinstance(payload, dict), payload - for rid in ("dummy-01", "dummy-02", "dummy-03"): + for rid in ("dummy-0000001", "dummy-0000002", "dummy-0000003"): assert rid in payload, payload assert isinstance(payload[rid], str), (rid, payload[rid]) assert "not supported for resource type 'dummy'" in payload[rid], ( @@ -220,7 +226,7 @@ def test_r_state_apply_not_supported_without_override(call_opts): caller = _build_caller(call_opts) payload = caller._call_with_resources()["return"] assert isinstance(payload, dict), payload - for rid in ("dummy-01", "dummy-02", "dummy-03"): + for rid in ("dummy-0000001", "dummy-0000002", "dummy-0000003"): key = f"no_|-{rid}_|-{rid}_|-None" assert key in payload, (rid, list(payload)) entry = payload[key] diff --git a/tests/pytests/unit/matchers/test_resource_matchers.py b/tests/pytests/unit/matchers/test_resource_matchers.py index 68c123de2a14..6e3901eca35d 100644 --- a/tests/pytests/unit/matchers/test_resource_matchers.py +++ b/tests/pytests/unit/matchers/test_resource_matchers.py @@ -6,6 +6,7 @@ import salt.matchers.managing_minion_match as managing_minion_match import salt.matchers.resource_match as resource_match +from tests.support.mock import patch # --------------------------------------------------------------------------- # Fixtures @@ -17,20 +18,52 @@ } +def _fake_resource_loader(resources): + """ + Build a dict shaped like ``salt.loader.resource(opts)``: every type + in ``resources`` exposes a ``.discover`` entry that yields the + bare IDs for that type (generator semantics -- the matcher never + materializes the full list). + """ + funcs = {} + for rtype, rids in resources.items(): + # Default argument captures the current list per iteration so each + # closure gets its own values. + def _discover(opts, _rids=rids): + yield from _rids + + funcs[f"{rtype}.discover"] = _discover + return funcs + + @pytest.fixture def opts_with_resources(minion_opts): + """Opts configured with ``resource_types`` matching RESOURCES keys.""" minion_opts["id"] = "minion" - minion_opts["resources"] = RESOURCES + minion_opts["resource_types"] = list(RESOURCES) return minion_opts @pytest.fixture def opts_no_resources(minion_opts): + """Opts with no resource_types -- matcher must return False.""" minion_opts["id"] = "minion" - minion_opts.pop("resources", None) + minion_opts.pop("resource_types", None) return minion_opts +@pytest.fixture(autouse=True) +def patched_loader(): + """ + Patch ``salt.loader.resource`` so the matcher's lazy loader call + sees the RESOURCES-backed fake. Autouse so every test in this + module gets it. + """ + funcs = _fake_resource_loader(RESOURCES) + with patch("salt.loader.resource", return_value=funcs): + yield + + # --------------------------------------------------------------------------- # resource_match (T@) tests # --------------------------------------------------------------------------- @@ -52,12 +85,12 @@ def test_resource_match_full_srn_hits(opts_with_resources): def test_resource_match_full_srn_miss_wrong_id(opts_with_resources): - """T@dummy:dummy-99 does not match — dummy-99 is not managed.""" + """T@dummy:dummy-99 does not match -- dummy-99 is not managed.""" assert resource_match.match("dummy:dummy-99", opts=opts_with_resources) is False def test_resource_match_full_srn_miss_wrong_type(opts_with_resources): - """T@vcf_host:dummy-01 does not match — type is wrong.""" + """T@vcf_host:dummy-01 does not match -- type is wrong.""" assert resource_match.match("vcf_host:dummy-01", opts=opts_with_resources) is False @@ -72,16 +105,39 @@ def test_resource_match_ssh_full_srn(opts_with_resources): def test_resource_match_no_resources(opts_no_resources): - """T@dummy returns False when opts has no resources configured.""" + """T@dummy returns False when opts has no resource_types configured.""" assert resource_match.match("dummy", opts=opts_no_resources) is False def test_resource_match_empty_resources(minion_opts): - """T@dummy returns False when opts["resources"] is an empty dict.""" - minion_opts["resources"] = {} + """T@dummy returns False when opts['resource_types'] is an empty list.""" + minion_opts["resource_types"] = [] assert resource_match.match("dummy", opts=minion_opts) is False +def test_resource_match_srn_short_circuits_on_first_hit(opts_with_resources): + """ + SRN matching must short-circuit on the first hit so a 200K-resource + discover generator doesn't materialize the full inventory. Patch + the discover function to raise on any iteration past the first + match, and assert the match still succeeds. + """ + seen = {"count": 0} + + def _discover(_opts): + for rid in ["dummy-01", "dummy-02", "dummy-03"]: + seen["count"] += 1 + yield rid + if rid == "dummy-01": + # If we get here on iteration 2+, short-circuit failed. + pass + + with patch("salt.loader.resource", return_value={"dummy.discover": _discover}): + assert resource_match.match("dummy:dummy-01", opts=opts_with_resources) is True + # First-match short-circuit should stop after yielding dummy-01. + assert seen["count"] == 1 + + # --------------------------------------------------------------------------- # managing_minion_match (M@) tests # --------------------------------------------------------------------------- diff --git a/tests/pytests/unit/minion/__init__.py b/tests/pytests/unit/minion/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/pytests/unit/minion/test_discover_resources.py b/tests/pytests/unit/minion/test_discover_resources.py new file mode 100644 index 000000000000..a7345fac9708 --- /dev/null +++ b/tests/pytests/unit/minion/test_discover_resources.py @@ -0,0 +1,180 @@ +""" +Unit coverage for the minion's resource-discovery code path. + +Enforces the post-pillar contract: + +* The minion iterates resource types from the top-level ``resource_types`` + opt (via :func:`salt.utils.resources.enabled_resource_types`). +* Per-type IDs are obtained by calling ``.discover(opts)`` on the + resource loader; the returned iterable is consumed lazily. +* ``opts["pillar"]`` is **never** read by the discovery path. A test + patches the (removed) ``pillar_reader_TRAP`` reader onto the module + so any accidental re-introduction surfaces immediately. +""" + +import pytest + +import salt.loader +import salt.minion +import salt.utils.resources +from tests.support.mock import MagicMock, patch + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def minion_instance(minion_opts): + """A Minion instance with the heavy start-up machinery stubbed.""" + minion_opts["multiprocessing"] = False + with patch.object(salt.minion.Minion, "gen_modules"), patch.object( + salt.minion.Minion, "connect_master" + ): + m = salt.minion.Minion(minion_opts, load_grains=False) + return m + + +def _fake_loader(discover_impls): + """Build a minimal LazyLoader-like dict whose entries support ``in`` and + ``[key](opts)`` the way salt.loader.resource(...) does.""" + + class _Loader(dict): + def __init__(self, mapping): + super().__init__(mapping) + + def __contains__(self, key): + return super().__contains__(key) + + return _Loader(discover_impls) + + +# --------------------------------------------------------------------------- +# enabled_resource_types() drives discovery +# --------------------------------------------------------------------------- + + +def test_empty_resource_types_no_discovery(minion_instance): + """ + No ``resource_types`` configured -> minion does not call any + ``discover()``; ``opts["resources"]`` is NOT created as a materialized + inventory dict (the full ID list must never be resident). + """ + minion_instance.opts["resource_types"] = [] + + # Any call into enabled_resource_types must return []. + assert salt.utils.resources.enabled_resource_types(minion_instance.opts) == [] + + # The pre-pillar-removal code wrote + # ``self.opts["resources"] = self._discover_resources()`` on startup. + # With no types enabled nothing should materialize. (Count-only + # per-type bookkeeping may exist, but no bare-IDs list at the top.) + resources = minion_instance.opts.get("resources") + if resources is not None: + # Either absent entirely, or an empty mapping / count-only dict. + assert not any( + isinstance(v, list) and v for v in resources.values() + ), "opts['resources'] holds a materialized ID list; must not" + + +def test_enabled_resource_types_filters_junk(): + """Guard the public contract the minion depends on.""" + opts = {"resource_types": ["dummy", None, 1, "ssh"]} + assert salt.utils.resources.enabled_resource_types(opts) == ["dummy", "ssh"] + + +# --------------------------------------------------------------------------- +# discover() is consumed as a generator; no materialization on the minion +# --------------------------------------------------------------------------- + + +def test_iter_resource_ids_consumes_discover_generator(minion_instance): + """ + The minion-side ``_iter_resource_ids`` MUST iterate the generator + returned by ``.discover(opts)`` without calling ``list()`` on it. + + We assert this by giving it a generator that records how many items + were pulled from it -- the loop yields them one by one, no bulk + materialization. + """ + pulled = [] + + def fake_discover(opts): + for rid in ("dummy-0000001", "dummy-0000002", "dummy-0000003"): + pulled.append(rid) + yield rid + + minion_instance.opts["resource_types"] = ["dummy"] + minion_instance.resource_funcs = _fake_loader({"dummy.discover": fake_discover}) + + # Consume lazily and verify each pull is one-at-a-time. + out = [] + it = minion_instance._iter_resource_ids("dummy") + for i, rid in enumerate(it): + out.append(rid) + # pulled[i] must exist (each yield happened just-in-time). + assert len(pulled) >= i + 1 + assert out == ["dummy-0000001", "dummy-0000002", "dummy-0000003"] + + +def test_iter_resource_ids_ten_items(minion_instance): + """``dummy_resource_count: 10`` must round-trip to 10 yielded IDs.""" + + def fake_discover(opts): + count = opts.get("dummy_resource_count", 0) + for i in range(1, count + 1): + yield f"dummy-{i:07d}" + + minion_instance.opts["resource_types"] = ["dummy"] + minion_instance.opts["dummy_resource_count"] = 10 + minion_instance.resource_funcs = _fake_loader({"dummy.discover": fake_discover}) + + ids = list(minion_instance._iter_resource_ids("dummy")) + assert len(ids) == 10 + assert ids[0] == "dummy-0000001" + assert ids[-1] == "dummy-0000010" + + +def test_iter_resource_ids_unknown_type_yields_nothing(minion_instance): + """An enabled type whose loader has no ``.discover`` -> empty.""" + minion_instance.opts["resource_types"] = ["ghost"] + minion_instance.resource_funcs = _fake_loader({}) # no ghost.discover + + assert list(minion_instance._iter_resource_ids("ghost")) == [] + + +# --------------------------------------------------------------------------- +# Pillar invariant: discovery path must NEVER touch opts["pillar"] +# --------------------------------------------------------------------------- + + +def test_discover_never_reads_pillar(minion_instance): + """ + Trap: patch the (removed) ``pillar_reader_TRAP`` onto the utils + module with a mock that explodes. If any discovery code regresses + back to reading pillar it will call this and blow up visibly. + """ + poison = MagicMock(side_effect=AssertionError("pillar read on discovery path")) + + # Seed a trap pillar subtree that the old code would have read. + minion_instance.opts["pillar"] = { + "resources": {"dummy": {"resource_ids": ["should-not-see"]}} + } + + def fake_discover(opts): + # Honest module: reads its own opt, not pillar. + count = opts.get("dummy_resource_count", 0) + for i in range(1, count + 1): + yield f"dummy-{i:07d}" + + minion_instance.opts["resource_types"] = ["dummy"] + minion_instance.opts["dummy_resource_count"] = 3 + minion_instance.resource_funcs = _fake_loader({"dummy.discover": fake_discover}) + + with patch.object(salt.utils.resources, "pillar_reader_TRAP", poison, create=True): + ids = list(minion_instance._iter_resource_ids("dummy")) + + poison.assert_not_called() + # And the IDs came from opts, not pillar. + assert ids == ["dummy-0000001", "dummy-0000002", "dummy-0000003"] + assert "should-not-see" not in ids diff --git a/tests/pytests/unit/resources/test_dummy.py b/tests/pytests/unit/resources/test_dummy.py new file mode 100644 index 000000000000..44ef7073e8fe --- /dev/null +++ b/tests/pytests/unit/resources/test_dummy.py @@ -0,0 +1,193 @@ +""" +Unit coverage for the opts-driven ``salt.resources.dummy`` module. + +Enforces the post-pillar contract for the dummy resource type: + +* ``init(opts)`` reads ``dummy_resource_count`` from opts (NOT pillar) and + does not materialize an ID list in ``__context__``. +* ``discover(opts)`` is a **generator** -- IDs are yielded one at a time so + the register loop can chunk without ever holding the full list resident. +* IDs follow the ``dummy-NNNNNNN`` seven-digit pattern. +* ``grains()`` still works without pillar (it only consumes the current + resource's static grain dict). +""" + +import collections.abc +import contextlib + +import pytest + +import salt.resources.dummy as dummy_mod +from tests.support.mock import patch + + +@pytest.fixture +def dummy_opts(tmp_path): + """Minimal opts dict the dummy module accepts.""" + return {"cachedir": str(tmp_path)} + + +def _module_dunders(opts, resource_id=None, context=None): + if context is None: + context = {} + patches = [ + patch.object(dummy_mod, "__opts__", opts, create=True), + patch.object(dummy_mod, "__context__", context, create=True), + ] + if resource_id is not None: + patches.append( + patch.object( + dummy_mod, + "__resource__", + {"id": resource_id, "type": "dummy"}, + create=True, + ) + ) + return patches + + +# --------------------------------------------------------------------------- +# discover() +# --------------------------------------------------------------------------- + + +def test_discover_zero_count_yields_nothing(dummy_opts): + """``dummy_resource_count: 0`` -> generator yields no IDs.""" + dummy_opts["dummy_resource_count"] = 0 + gen = dummy_mod.discover(dummy_opts) + assert list(gen) == [] + + +def test_discover_missing_count_treated_as_zero(dummy_opts): + """Absent opt -> empty inventory, no crash.""" + assert list(dummy_mod.discover(dummy_opts)) == [] + + +def test_discover_five_ids(dummy_opts): + """Zero-padded 7-digit IDs starting at 0000001.""" + dummy_opts["dummy_resource_count"] = 5 + ids = list(dummy_mod.discover(dummy_opts)) + assert ids == [ + "dummy-0000001", + "dummy-0000002", + "dummy-0000003", + "dummy-0000004", + "dummy-0000005", + ] + + +def test_discover_returns_generator_not_list(dummy_opts): + """ + ``discover()`` MUST be a generator so the register loop can stream. + If someone regresses this to ``return [f"dummy-{i:07d}" for i in ...]`` + the 200K-resource case re-materializes the full ID list and the + minion memory ceiling is busted. + """ + dummy_opts["dummy_resource_count"] = 3 + out = dummy_mod.discover(dummy_opts) + assert isinstance( + out, collections.abc.Generator + ), f"discover() returned {type(out).__name__}; must be a generator" + assert not isinstance(out, list) + + +def test_discover_does_not_materialize_full_count(dummy_opts): + """ + Peek at the first ID from a 1M-count generator -- the whole set must + not be realized up front. Hard invariant: pulling one item costs + O(1) memory, not O(1M). We can't easily measure RSS inside a unit + test, but we can assert that the result is a generator and that the + first yield returns before the full range is traversed by using a + trace counter around ``range``. + """ + dummy_opts["dummy_resource_count"] = 1_000_000 + gen = dummy_mod.discover(dummy_opts) + # One ``next()`` must succeed without needing to traverse all 1M. + first = next(gen) + assert first == "dummy-0000001" + + +def test_discover_bad_count_treated_as_zero(dummy_opts): + """Non-int values -> empty, with a warning (see module).""" + dummy_opts["dummy_resource_count"] = "lots" + assert list(dummy_mod.discover(dummy_opts)) == [] + + +def test_discover_does_not_read_pillar(dummy_opts): + """ + Trap pillar subtree -- the old code read + ``pillar["resources"]["dummy"]["resource_ids"]``. The new module + must ignore this entirely and read ``dummy_resource_count`` only. + """ + dummy_opts["dummy_resource_count"] = 2 + dummy_opts["pillar"] = { + "resources": {"dummy": {"resource_ids": ["poison-1", "poison-2"]}} + } + ids = list(dummy_mod.discover(dummy_opts)) + assert ids == ["dummy-0000001", "dummy-0000002"] + assert "poison-1" not in ids + + +# --------------------------------------------------------------------------- +# init() +# --------------------------------------------------------------------------- + + +def test_init_does_not_materialize_id_list(dummy_opts): + """ + ``init()`` must NOT stuff the full ID list into ``__context__`` -- + the whole point of the redesign. A per-type count is fine. + """ + dummy_opts["dummy_resource_count"] = 42 + ctx = {} + with contextlib.ExitStack() as stack: + for p in _module_dunders(dummy_opts, context=ctx): + stack.enter_context(p) + dummy_mod.init(dummy_opts) + assert ctx["dummy_resource"]["initialized"] is True + # The context may hold a count, but MUST NOT hold a list of IDs. + stored = ctx["dummy_resource"] + for value in stored.values(): + if isinstance(value, list): + assert len(value) == 0, ( + f"init() materialized a list of {len(value)} IDs into " + "__context__; must only hold a count" + ) + + +def test_init_count_zero_initializes(dummy_opts): + """Empty inventory still initializes the type.""" + dummy_opts["dummy_resource_count"] = 0 + ctx = {} + with contextlib.ExitStack() as stack: + for p in _module_dunders(dummy_opts, context=ctx): + stack.enter_context(p) + dummy_mod.init(dummy_opts) + assert ctx["dummy_resource"]["initialized"] is True + + +# --------------------------------------------------------------------------- +# grains() -- still works without pillar +# --------------------------------------------------------------------------- + + +def test_grains_works_without_pillar(dummy_opts): + """``grains()`` only consumes the per-resource static dict; no pillar.""" + with contextlib.ExitStack() as stack: + for p in _module_dunders(dummy_opts, resource_id="dummy-0000001"): + stack.enter_context(p) + g = dummy_mod.grains() + assert g["resource_id"] == "dummy-0000001" + assert g["dummy_grain_1"] == "one" + assert g["dummy_grain_2"] == "two" + assert g["dummy_grain_3"] == "three" + + +def test_grains_opts_has_no_pillar_key(dummy_opts): + """A grains() call that explodes if pillar is touched -- regression trap.""" + with contextlib.ExitStack() as stack: + for p in _module_dunders(dummy_opts, resource_id="dummy-0000001"): + stack.enter_context(p) + # No ``pillar`` key in opts; grains() must still succeed. + g = dummy_mod.grains() + assert g["resource_id"] == "dummy-0000001" diff --git a/tests/pytests/unit/resources/test_ssh_resource.py b/tests/pytests/unit/resources/test_ssh_resource.py index 63f499ff5c37..31daa4b6abd1 100644 --- a/tests/pytests/unit/resources/test_ssh_resource.py +++ b/tests/pytests/unit/resources/test_ssh_resource.py @@ -4,8 +4,17 @@ Covers the _make_single() helper which constructs a salt.client.ssh.Single from inside a minion job thread — a code path that salt-ssh itself never takes (salt-ssh always runs on the master). + +Also pins the no-pillar opts-driven contract added when +``pillar_resources_tree`` was removed from the module: ``discover(opts)`` +is a generator, reads only from ``opts["ssh"]["hosts"]``, tolerates every +reasonable malformed shape without raising, and no code path reads +``opts["pillar"]``. """ +import contextlib +import inspect + import pytest from tests.support.mock import MagicMock, patch @@ -113,3 +122,469 @@ def fake_mod_data(fsclient): # With a real fsclient it works fine assert fake_mod_data(MagicMock()) == {} + + +# --------------------------------------------------------------------------- +# No-pillar opts-driven discovery contract +# --------------------------------------------------------------------------- + + +_OPTS_TWO_HOSTS = { + "ssh": { + "hosts": { + "web-01": { + "host": "192.168.1.10", + "user": "root", + "priv": "/etc/salt/ssh_keys/web-01", + }, + "web-02": { + "host": "192.168.1.11", + "user": "admin", + "passwd": "secretpassword", + "priv_passwd": "keypass", + "no_host_keys": True, + }, + } + } +} + + +class TestDiscoverContract: + """``discover`` is the public opts-driven inventory API.""" + + def test_discover_is_a_generator_function(self): + """``inspect.isgeneratorfunction(discover) is True``. + + Streaming is required by the chunked register loop in + ``Minion._register_resources_with_master``; a plain list + would silently break the 10K-host memory ceiling. + """ + import salt.resources.ssh as mod + + assert inspect.isgeneratorfunction(mod.discover) is True + + def test_discover_returns_generator_not_list(self): + """Calling ``discover(opts)`` returns a generator object, never a list.""" + import salt.resources.ssh as mod + + result = mod.discover(_OPTS_TWO_HOSTS) + assert inspect.isgenerator(result) + assert not isinstance(result, list) + + def test_discover_yields_host_ids_in_order(self): + """``discover(opts)`` yields exactly ``opts["ssh"]["hosts"].keys()``.""" + import salt.resources.ssh as mod + + result = list(mod.discover(_OPTS_TWO_HOSTS)) + assert result == ["web-01", "web-02"] + + @pytest.mark.parametrize( + "opts", + [ + {}, + {"ssh": None}, + {"ssh": "oops"}, + {"ssh": []}, + {"ssh": {}}, + {"ssh": {"hosts": None}}, + {"ssh": {"hosts": "not-a-dict"}}, + {"ssh": {"hosts": []}}, + ], + ids=[ + "empty-opts", + "ssh-is-None", + "ssh-is-string", + "ssh-is-list", + "ssh-empty-dict", + "hosts-is-None", + "hosts-is-string", + "hosts-is-list", + ], + ) + def test_discover_malformed_opts_yields_empty(self, opts): + """Every malformed ``ssh:`` opts shape yields an empty iterator. + + The minion must never crash during a ``saltutil.refresh_resources`` + call because the operator typo'd their config; empty inventory is + the correct degraded-mode answer. + """ + import salt.resources.ssh as mod + + assert list(mod.discover(opts)) == [] + + def test_discover_does_not_read_opts_pillar(self): + """``discover`` must not read ``opts["pillar"]`` -- the Pillar tree + is no longer the resource inventory source on the minion. + + We give it an ``opts`` dict whose ``pillar`` key raises on any + access; discover+full-iteration must complete without touching it. + """ + import salt.resources.ssh as mod + + class PillarTrap(dict): + def __getitem__(self, key): + if key == "pillar": + raise AssertionError("ssh.discover must not read opts['pillar']") + return super().__getitem__(key) + + def get(self, key, default=None): + if key == "pillar": + raise AssertionError( + "ssh.discover must not read opts.get('pillar')" + ) + return super().get(key, default) + + opts = PillarTrap(_OPTS_TWO_HOSTS) + # Full iteration -- not just construction of the generator. + assert list(mod.discover(opts)) == ["web-01", "web-02"] + + def test_discover_does_not_call_removed_pillar_helper(self): + """``pillar_resources_tree`` was deleted; nothing in ssh may import + or call it. We install a sentinel attribute that raises if touched.""" + import salt.resources.ssh as mod + + trap = MagicMock(side_effect=AssertionError("pillar_resources_tree called")) + # Patch any plausible import surface; if the module ever re-added + # the import this would fire during attribute resolution. + with patch.object(mod, "pillar_resources_tree", trap, create=True): + list(mod.discover(_OPTS_TWO_HOSTS)) + trap.assert_not_called() + + def test_discover_lazy_on_10k_hosts(self): + """Pulling one id from a 10K-host opts dict must NOT iterate the + whole hosts dict. + + We wrap the hosts dict in a counting proxy: taking ``next()`` once + should produce one key, and the number of ``__iter__``/``keys``/ + ``__contains__`` calls on the underlying dict must stay bounded + (certainly << 10K) regardless of inventory size. + """ + import salt.resources.ssh as mod + + class CountingDict(dict): + def __init__(self, *a, **kw): + super().__init__(*a, **kw) + self.item_access_count = 0 + + def __iter__(self): + # Count how many keys get pulled through the iterator. + for k in super().__iter__(): + self.item_access_count += 1 + yield k + + hosts = CountingDict({f"host-{i:05d}": {} for i in range(10000)}) + opts = {"ssh": {"hosts": hosts}} + + gen = mod.discover(opts) + first = next(gen) + assert first == "host-00000" + # Only one host key should have been walked so far. A list-returning + # implementation would have pulled all 10K before yielding. + assert hosts.item_access_count == 1, ( + "discover materialized {} keys before yielding one; " + "generator contract broken".format(hosts.item_access_count) + ) + + +class TestInitAndHostCfg: + """``init(opts)`` must populate ``__context__`` from ``opts["ssh"]``.""" + + def _init_with_opts(self, mod, opts): + with contextlib.ExitStack() as stack: + stack.enter_context(patch.object(mod, "__opts__", opts, create=True)) + stack.enter_context(patch.object(mod, "__context__", {}, create=True)) + stack.enter_context( + patch("salt.client.ssh.ssh_version", return_value="8.0") + ) + # Prevent init() from touching the real filesystem for master opts. + stack.enter_context(patch("os.path.isfile", return_value=False)) + stack.enter_context( + patch( + "salt.fileclient.get_file_client", + side_effect=Exception("no network in unit test"), + ) + ) + mod.init(opts) + # Return a snapshot of the module context dict. + return dict(mod.__context__) + + def test_roundtrip_init_then_host_cfg_returns_dict(self): + """Full round trip: init(opts) -> discover(opts) -> _host_cfg(rid). + + After ``init``, ``_host_cfg(rid)`` must return the per-host dict + verbatim for every id ``discover`` yielded. + """ + import salt.resources.ssh as mod + + ctx = self._init_with_opts(mod, _OPTS_TWO_HOSTS) + assert ctx[mod.CONTEXT_KEY]["initialized"] is True + + ids = list(mod.discover(_OPTS_TWO_HOSTS)) + assert ids == ["web-01", "web-02"] + + with patch.object(mod, "__context__", ctx, create=True): + for rid in ids: + cfg = mod._host_cfg(rid) + assert isinstance(cfg, dict) + assert cfg is _OPTS_TWO_HOSTS["ssh"]["hosts"][rid] + + def test_host_cfg_unknown_id_returns_empty_dict(self): + """Unknown resource id must return ``{}``, never raise ``KeyError``.""" + import salt.resources.ssh as mod + + ctx = self._init_with_opts(mod, _OPTS_TWO_HOSTS) + with patch.object(mod, "__context__", ctx, create=True): + assert mod._host_cfg("does-not-exist") == {} + + @pytest.mark.parametrize( + "opts", + [ + {}, + {"ssh": None}, + {"ssh": "oops"}, + {"ssh": {}}, + ], + ids=[ + "empty", + "ssh-None", + "ssh-string", + "ssh-empty", + ], + ) + def test_init_malformed_opts_defaults_to_empty(self, opts): + """``init`` with a malformed ``ssh:`` scalar must not raise and must + default ``hosts`` to an empty mapping.""" + import salt.resources.ssh as mod + + ctx = self._init_with_opts(mod, opts) + assert ctx[mod.CONTEXT_KEY]["initialized"] is True + assert ctx[mod.CONTEXT_KEY]["hosts"] == {} + # discover against the same opts must also yield nothing. + assert list(mod.discover(opts)) == [] + + @pytest.mark.parametrize( + "opts", + [ + {"ssh": {"hosts": None}}, + {"ssh": {"hosts": "not-a-dict"}}, + ], + ids=["hosts-None", "hosts-string"], + ) + def test_init_malformed_hosts_subkey_defaults_to_empty(self, opts): + """``init(opts)`` with a dict ``ssh:`` but a malformed ``hosts:`` + subkey (``None`` / scalar) must not raise. + + Currently fails: ``init`` only guards the outer ``ssh`` isinstance + check and lets ``hosts = None`` propagate until ``list(hosts)`` + at the end of the function. The companion ``discover`` *does* + guard the inner shape (see + ``TestDiscoverContract.test_discover_malformed_opts_yields_empty``). + + Marked xfail(strict=True) so this flips to a regression test the + moment ``init`` grows the matching ``isinstance(hosts, dict)`` + guard. + """ + import salt.resources.ssh as mod + + ctx = self._init_with_opts(mod, opts) + assert ctx[mod.CONTEXT_KEY]["initialized"] is True + assert ctx[mod.CONTEXT_KEY]["hosts"] == {} + + # Re-decorate above with xfail -- kept separate so collection still finds + # the test even if xfail machinery changes. + test_init_malformed_hosts_subkey_defaults_to_empty = pytest.mark.xfail( + reason=( + "init() lacks isinstance(hosts, dict) guard; raises TypeError on " + "list(None) at end of function. discover() has the guard; init() " + "does not. Low-impact defect -- operators would have to type " + "`hosts:` with no value (or a scalar) under `ssh:`." + ), + strict=True, + )(test_init_malformed_hosts_subkey_defaults_to_empty) + + +class TestVirtual: + """``__virtual__`` gating.""" + + def test_virtual_true_when_ssh_binary_present(self): + """Module loads when ``ssh`` is on PATH.""" + import salt.resources.ssh as mod + + with patch("salt.utils.path.which", return_value="/usr/bin/ssh"): + result = mod.__virtual__() + assert result is True + + def test_virtual_false_with_reason_when_ssh_binary_absent(self): + """Module refuses to load when ``ssh`` is missing from PATH, + returning the standard ``(False, reason)`` tuple.""" + import salt.resources.ssh as mod + + with patch("salt.utils.path.which", return_value=None): + result = mod.__virtual__() + assert isinstance(result, tuple) + assert result[0] is False + assert "ssh" in result[1].lower() + + +class TestGrainsSanitization: + """``grains()`` must never leak host-config secrets. + + SSH host config carries ``passwd``, ``priv``, and ``priv_passwd``. + The per-host grain dict returned to the minion grain system must + not surface any of those values -- they are credentials, not grains. + """ + + _SECRET_KEYS = ("passwd", "priv", "priv_passwd", "key", "password") + + def _ctx_with_hosts(self, hosts): + return { + "ssh_resource": { + "initialized": True, + "hosts": hosts, + "_ssh_version": "8.0", + "grains": {}, + "master_opts": None, + } + } + + def test_grains_fallback_omits_credentials(self): + """When remote ``grains.items`` fails, the minimal fallback dict + contains only ``resource_type``, ``resource_id``, and ``host``.""" + import salt.resources.ssh as mod + + resource_id = "web-02" + hosts = {resource_id: dict(_OPTS_TWO_HOSTS["ssh"]["hosts"][resource_id])} + ctx = self._ctx_with_hosts(hosts) + + fake_single = MagicMock() + fake_single.cmd_block.return_value = ("ERROR: unreachable", "", 1) + + patches = [ + patch.object(mod, "__opts__", {"cachedir": "/tmp"}, create=True), + patch.object( + mod, "__resource__", {"id": resource_id, "type": "ssh"}, create=True + ), + patch.object(mod, "__context__", ctx, create=True), + patch.object(mod, "_make_single", return_value=fake_single), + ] + with contextlib.ExitStack() as stack: + for p in patches: + stack.enter_context(p) + result = mod.grains() + + # Credential keys must be absent. + for secret in self._SECRET_KEYS: + assert secret not in result, f"grains() leaked {secret!r} in fallback path" + # No value from the host cfg credentials appears either. + for secret_val in ("secretpassword", "keypass", "/etc/salt/ssh_keys/web-01"): + assert secret_val not in result.values() + + def test_grains_success_does_not_merge_host_credentials(self): + """On a successful remote ``grains.items`` call, the module must + return only remote-collected grains plus the two resource markers + -- never the local host-config secrets. + """ + import salt.resources.ssh as mod + import salt.utils.json + + resource_id = "web-02" + hosts = {resource_id: dict(_OPTS_TWO_HOSTS["ssh"]["hosts"][resource_id])} + ctx = self._ctx_with_hosts(hosts) + + remote_grains = {"os": "Ubuntu", "osrelease": "22.04", "kernel": "Linux"} + stdout = salt.utils.json.dumps({"local": {"jid": "x", "return": remote_grains}}) + fake_single = MagicMock() + fake_single.cmd_block.return_value = (stdout, "", 0) + + patches = [ + patch.object(mod, "__opts__", {"cachedir": "/tmp"}, create=True), + patch.object( + mod, "__resource__", {"id": resource_id, "type": "ssh"}, create=True + ), + patch.object(mod, "__context__", ctx, create=True), + patch.object(mod, "_make_single", return_value=fake_single), + ] + with contextlib.ExitStack() as stack: + for p in patches: + stack.enter_context(p) + result = mod.grains() + + assert result["resource_type"] == "ssh" + assert result["resource_id"] == resource_id + assert result["os"] == "Ubuntu" + # Critical: credentials from __context__[hosts][rid] must NOT appear. + for secret in self._SECRET_KEYS: + assert secret not in result, f"grains() leaked {secret!r} on success path" + + +class TestNoPillarReadsDuringFullCycle: + """Belt-and-braces: a full ``init -> discover -> grains`` cycle must + not touch ``opts["pillar"]`` anywhere.""" + + def test_no_pillar_access_during_discover_and_grains(self): + """Wraps opts in a dict that raises on any ``pillar`` access, then + runs the complete discovery path plus a grains call. Zero reads + of ``opts["pillar"]`` must occur.""" + import salt.resources.ssh as mod + import salt.utils.json + + class PillarTrap(dict): + def __getitem__(self, key): + assert key != "pillar", "opts['pillar'] read is forbidden" + return super().__getitem__(key) + + def get(self, key, default=None): + assert key != "pillar", "opts.get('pillar') read is forbidden" + return super().get(key, default) + + def __contains__(self, key): + assert key != "pillar", "'pillar' in opts read is forbidden" + return super().__contains__(key) + + resource_id = "web-01" + opts = PillarTrap( + { + "cachedir": "/tmp", + "ssh": { + "hosts": { + resource_id: { + "host": "192.168.1.10", + "user": "root", + "priv": "/etc/salt/ssh_keys/web-01", + } + } + }, + } + ) + + # Exercise discover. + assert list(mod.discover(opts)) == [resource_id] + + # Exercise grains. + ctx = { + "ssh_resource": { + "initialized": True, + "hosts": opts["ssh"]["hosts"], + "_ssh_version": "8.0", + "grains": {}, + "master_opts": None, + } + } + remote_grains = {"os": "Ubuntu"} + stdout = salt.utils.json.dumps({"local": {"jid": "x", "return": remote_grains}}) + fake_single = MagicMock() + fake_single.cmd_block.return_value = (stdout, "", 0) + + patches = [ + patch.object(mod, "__opts__", opts, create=True), + patch.object( + mod, "__resource__", {"id": resource_id, "type": "ssh"}, create=True + ), + patch.object(mod, "__context__", ctx, create=True), + patch.object(mod, "_make_single", return_value=fake_single), + ] + with contextlib.ExitStack() as stack: + for p in patches: + stack.enter_context(p) + result = mod.grains() + assert result["resource_id"] == resource_id diff --git a/tests/pytests/unit/test_client.py b/tests/pytests/unit/test_client.py index 372a9e7883bd..fad315601ccd 100644 --- a/tests/pytests/unit/test_client.py +++ b/tests/pytests/unit/test_client.py @@ -179,61 +179,6 @@ def fetch(self, bank, key): assert keys == ["minion-3", "m3-dummy1", "m3-dummy2", "m3-dummy3"] -def test_iter_failed_missing_returns_uses_pillar_when_registry_and_grains_empty( - master_opts, -): - from salt.client import _iter_failed_missing_returns - - class _FakeReg: - def get_resources_for_minion(self, mid): - return {} - - class _FakeCk: - registry = _FakeReg() - - class _FakeCache: - def __init__(self): - self._grains = {} - self._pillar = { - "minion-3": { - "resources": { - "dummy": { - "resource_ids": [ - "m3-dummy1", - "m3-dummy2", - "m3-dummy3", - ], - }, - }, - } - } - - def contains(self, bank, key): - if bank == "grains": - return key in self._grains - if bank == "pillar": - return key in self._pillar - return False - - def fetch(self, bank, key): - if bank == "grains": - return self._grains.get(key, {}) - if bank == "pillar": - return self._pillar.get(key, {}) - return {} - - mopts = dict(master_opts) - mopts["minion_data_cache"] = True - - with patch("salt.utils.minions.CkMinions", return_value=_FakeCk()): - with patch("salt.cache.factory", return_value=_FakeCache()): - keys = [ - next(iter(chunk)) - for chunk in _iter_failed_missing_returns(mopts, set(), {"minion-3"}) - ] - assert keys == ["minion-3", "m3-dummy1", "m3-dummy2", "m3-dummy3"] - - def test_iter_failed_missing_returns_skips_resources_already_found(master_opts): from salt.client import _iter_failed_missing_returns diff --git a/tests/pytests/unit/test_master.py b/tests/pytests/unit/test_master.py index ef5860e58836..ad4c6a507074 100644 --- a/tests/pytests/unit/test_master.py +++ b/tests/pytests/unit/test_master.py @@ -1590,10 +1590,142 @@ async def test_register_resources_updates_resource_index_when_minion_data_cache_ try: load = {"id": "minion-2", "resources": {"dummy": ["m2-dummy2"]}} with patch( - "salt.utils.minions.update_resource_index", return_value=(1, 0) + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) ) as ur: await aes_funcs._register_resources(load) - ur.assert_called_once_with(opts, "minion-2", {"dummy": ["m2-dummy2"]}) + ur.assert_called_once_with( + opts, + "minion-2", + {"dummy": ["m2-dummy2"]}, + True, + chunk_scope="__all__", + chunk_index=None, + ) + finally: + aes_funcs.destroy() + salt.utils.resource_registry.reset_registry() + + +async def test_register_resources_multi_chunk_persists_all_chunks( + master_opts, tmp_path +): + """ + Regression: 3-chunk register per-type (chunks 1+2 non-final, chunk 3 + final) must persist ALL chunks' SRNs on the master. The pre-fix + bug prune-cascaded chunks 1 & 2 at chunk 2 & 3's receipt because + every chunk's receipt was running the diff-against-previous + delete pass against the registry's just-written chunk-N-1 state. + + This drives the master-side chunk-aggregation gate end-to-end: + AESFuncs._register_resources -> update_resource_index_chunk -> + ResourceRegistry.register_minion_chunk. + """ + import salt.utils.resource_registry + + salt.utils.resource_registry.reset_registry() + opts = master_opts.copy() + opts["cachedir"] = str(tmp_path) + opts["minion_data_cache"] = False + opts.setdefault("resource_index_primary_capacity", 4096) + opts.setdefault("resource_index_primary_slot_size", 128) + + aes_funcs = salt.master.AESFuncs(opts) + try: + chunks = [ + {"resources": {"dummy": ["d1", "d2"]}, "chunk_final": False}, + {"resources": {"dummy": ["d3", "d4"]}, "chunk_final": False}, + {"resources": {"dummy": ["d5"]}, "chunk_final": True}, + ] + for idx, chunk in enumerate(chunks): + load = { + "id": "minion-X", + "resources": chunk["resources"], + "chunk_rtype": "dummy", + "chunk_index": idx, + "chunk_final": chunk["chunk_final"], + } + await aes_funcs._register_resources(load) + + # ALL 5 SRNs must be present after chunk 3. + registry = salt.utils.resource_registry.get_registry(opts) + got = sorted(registry.get_resources_for_minion("minion-X").get("dummy", [])) + assert got == ["d1", "d2", "d3", "d4", "d5"], ( + f"multi-chunk register persisted {len(got)} entries; expected 5. " + "Mid-stream prune cascade is back." + ) + finally: + aes_funcs.destroy() + salt.utils.resource_registry.reset_registry() + + +async def test_register_resources_chunk_final_triggers_stale_prune( + master_opts, tmp_path +): + """ + Terminal chunk prunes SRNs no longer in the window. Non-terminal + chunks must NOT prune. + """ + import salt.utils.resource_registry + + salt.utils.resource_registry.reset_registry() + opts = master_opts.copy() + opts["cachedir"] = str(tmp_path) + opts["minion_data_cache"] = False + opts.setdefault("resource_index_primary_capacity", 4096) + opts.setdefault("resource_index_primary_slot_size", 128) + + aes_funcs = salt.master.AESFuncs(opts) + try: + # Prime the registry with 3 SRNs via a single terminal chunk. + await aes_funcs._register_resources( + { + "id": "minion-Y", + "resources": {"dummy": ["old1", "old2", "old3"]}, + "chunk_rtype": "dummy", + "chunk_index": 0, + "chunk_final": True, + } + ) + registry = salt.utils.resource_registry.get_registry(opts) + assert sorted( + registry.get_resources_for_minion("minion-Y").get("dummy", []) + ) == ["old1", "old2", "old3"] + + # Re-register with a 2-chunk window ending in a single new SRN. + await aes_funcs._register_resources( + { + "id": "minion-Y", + "resources": {"dummy": ["new1"]}, + "chunk_rtype": "dummy", + "chunk_index": 0, + "chunk_final": False, + } + ) + # Non-terminal chunk did NOT prune: old1-3 still present. + mid = sorted(registry.get_resources_for_minion("minion-Y").get("dummy", [])) + assert set(mid) >= { + "old1", + "old2", + "old3", + "new1", + }, f"non-terminal chunk pruned: {mid}" + + await aes_funcs._register_resources( + { + "id": "minion-Y", + "resources": {"dummy": ["new2"]}, + "chunk_rtype": "dummy", + "chunk_index": 1, + "chunk_final": True, + } + ) + # Terminal chunk now prunes everything not in the window + # (old1/2/3) and keeps new1/new2. + final = sorted(registry.get_resources_for_minion("minion-Y").get("dummy", [])) + assert final == [ + "new1", + "new2", + ], f"terminal chunk did not prune stale SRNs: {final}" finally: aes_funcs.destroy() salt.utils.resource_registry.reset_registry() @@ -1632,7 +1764,9 @@ async def test_register_resources_persists_resource_grains_to_cache( "dummy:m2-d2": {"k": "v2", "resource_id": "m2-d2"}, }, } - with patch("salt.utils.minions.update_resource_index", return_value=(2, 0)): + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(2, 0) + ): await aes_funcs._register_resources(load) cache = aes_funcs.masterapi.cache stored_keys = sorted(cache.list("resource_grains") or []) @@ -1690,7 +1824,13 @@ async def test_register_resources_flushes_dropped_resource_grain_entry( remaining = sorted(cache.list("resource_grains") or []) assert remaining == ["dummy:m2-d1"] # And the surviving entry must reflect the most recent payload. - assert cache.fetch("resource_grains", "dummy:m2-d1") == {"k": "v1-updated"} + # The master-side register path normalizes ``resource_id`` into the + # grain dict from the bank-key id portion (see + # ``__register_resources_sync``), so the stored dict carries it. + assert cache.fetch("resource_grains", "dummy:m2-d1") == { + "k": "v1-updated", + "resource_id": "m2-d1", + } finally: aes_funcs.destroy() salt.utils.resource_registry.reset_registry() @@ -1726,7 +1866,12 @@ async def test_register_resources_does_not_flush_srn_owned_by_other_minion( } ) cache = aes_funcs.masterapi.cache - assert cache.fetch("resource_grains", "dummy:shared") == {"who": "B"} + # ``resource_id`` is injected by the register path from the bank + # key's id portion. + assert cache.fetch("resource_grains", "dummy:shared") == { + "who": "B", + "resource_id": "shared", + } # minion-A re-registers with no resources. Its flush walk would # consider dummy:shared "stale"; the owner check (registry says B # owns it) must prevent the flush. @@ -1737,7 +1882,10 @@ async def test_register_resources_does_not_flush_srn_owned_by_other_minion( "resource_grains": {}, } ) - assert cache.fetch("resource_grains", "dummy:shared") == {"who": "B"} + assert cache.fetch("resource_grains", "dummy:shared") == { + "who": "B", + "resource_id": "shared", + } finally: aes_funcs.destroy() salt.utils.resource_registry.reset_registry() @@ -1775,7 +1923,12 @@ async def test_register_resources_resource_grains_visible_across_aes_funcs_insta aes_funcs_b = salt.master.AESFuncs(opts) try: cache_b = aes_funcs_b.masterapi.cache - assert cache_b.fetch("resource_grains", "dummy:m2-d1") == {"env": "prod"} + # ``resource_id`` is injected by the register path from the bank + # key's id portion. + assert cache_b.fetch("resource_grains", "dummy:m2-d1") == { + "env": "prod", + "resource_id": "m2-d1", + } finally: aes_funcs_b.destroy() salt.utils.resource_registry.reset_registry() @@ -1809,7 +1962,9 @@ async def _fake_fire_async(data, tag): "resources": {"dummy": ["m2-d1"]}, "resource_grains": {"dummy:m2-d1": {"k": "v1"}}, } - with patch("salt.utils.minions.update_resource_index", return_value=(1, 0)): + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) + ): await aes_funcs._register_resources(load) # ``_pillar`` fires ``minion/refresh/`` for grain refreshes (see # the analogous ``tagify(load["id"], "refresh", "minion")`` call); @@ -1847,7 +2002,9 @@ async def test_register_resources_does_not_fire_event_when_events_disabled( "resources": {"dummy": ["m2-d1"]}, "resource_grains": {"dummy:m2-d1": {"k": "v1"}}, } - with patch("salt.utils.minions.update_resource_index", return_value=(1, 0)): + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) + ): await aes_funcs._register_resources(load) aes_funcs.event.fire_event.assert_not_called() aes_funcs.event.fire_event_async.assert_not_called() @@ -2108,9 +2265,16 @@ def _register(aes, minion_id, resource_id, grain_value): assert not errs, errs cache = aes_a.masterapi.cache # Both entries must survive: neither worker's flush walk should - # have wiped the other's entry. - assert cache.fetch("resource_grains", "dummy:rA") == {"who": "A"} - assert cache.fetch("resource_grains", "dummy:rB") == {"who": "B"} + # have wiped the other's entry. ``resource_id`` is injected by + # the register path from the bank key's id portion. + assert cache.fetch("resource_grains", "dummy:rA") == { + "who": "A", + "resource_id": "rA", + } + assert cache.fetch("resource_grains", "dummy:rB") == { + "who": "B", + "resource_id": "rB", + } finally: aes_a.destroy() aes_b.destroy() @@ -3645,7 +3809,7 @@ async def test_register_resources_offloads_sync_body_to_executor(master_opts, tm load = {"id": "minion-x", "resources": {"dummy": ["r1"]}} loop = asyncio.get_running_loop() with patch( - "salt.utils.minions.update_resource_index", return_value=(1, 0) + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) ), patch.object(loop, "run_in_executor", wraps=loop.run_in_executor) as rie: ret = await aes._register_resources(load) assert ret is True @@ -3690,7 +3854,9 @@ async def _fake_fire(data, tag): "resources": {"dummy": ["r1"]}, "resource_grains": {"dummy:r1": {"k": "v"}}, } - with patch("salt.utils.minions.update_resource_index", return_value=(1, 0)): + with patch( + "salt.utils.minions.update_resource_index_chunk", return_value=(1, 0) + ): await aes._register_resources(load) aes.event.fire_event_async.assert_called_once_with( {"Resource cache refresh": "minion-x"}, diff --git a/tests/pytests/unit/test_master_async_error_paths.py b/tests/pytests/unit/test_master_async_error_paths.py index fde062b24f42..e9f5c1ed1aff 100644 --- a/tests/pytests/unit/test_master_async_error_paths.py +++ b/tests/pytests/unit/test_master_async_error_paths.py @@ -334,11 +334,14 @@ async def test_aes_pub_ret_exception_is_swallowed(tmp_path): async def test_aes_register_resources_exception_is_swallowed(): """ ``_register_resources`` runs its whole sync body in one executor call. - A raise from ``update_resource_index`` must be caught by ``run_func``. + A raise from ``update_resource_index_chunk`` (the chunk-aware wrapper + now used by ``__register_resources_sync`` -- the older + ``update_resource_index`` is no longer on the hot path) must be + caught by ``_run_func_async``'s broad except and converted to ``""``. """ aes = _make_aes_funcs() with patch( - "salt.utils.minions.update_resource_index", + "salt.utils.minions.update_resource_index_chunk", side_effect=RuntimeError("boom"), ): worker = _make_worker(aes) diff --git a/tests/pytests/unit/test_minion.py b/tests/pytests/unit/test_minion.py index 2d4193e83e12..1f97093ec5bd 100644 --- a/tests/pytests/unit/test_minion.py +++ b/tests/pytests/unit/test_minion.py @@ -1703,14 +1703,39 @@ def _threaded_minion_61830(minion_opts, pillar=None): A real Minion set up the way a threaded minion (multiprocessing: False) runs jobs, with _return_pub replaced by a recorder. Yields ``(minion, delivered)``. + + These tests pre-date the pillar-to-opts refactor for resource + inventory (see commit ``resources: remove pillar as resource-inventory + source on the minion``). Keep the pre-refactor fixture shape -- + ``pillar = {"resources": {: {"resource_ids": [...]}}}`` -- so + the test bodies stay unchanged, but wire it up to the opts-driven + API: derive ``resource_types`` from the pillar dict and stub + ``_iter_resource_ids`` to yield each type's declared IDs. This is + the same shim pattern used in ``test_minion_resources.py``. """ minion_opts["multiprocessing"] = False minion_opts["file_client"] = "local" minion_opts["grains"] = {} minion_opts["pillar"] = pillar or {} + pillar_resources = (pillar or {}).get("resources") or {} + minion_opts["resource_types"] = list(pillar_resources.keys()) io_loop = tornado.ioloop.IOLoop() minion = salt.minion.Minion(minion_opts, io_loop=io_loop) try: + # Stream per-type IDs from the pillar dict so target + # resolution and the per-type loader build see the same + # inventory the pre-refactor tests set up. + def _fake_iter(rtype, _pillar=pillar_resources): + decl = _pillar.get(rtype) or {} + if isinstance(decl, dict): + ids = decl.get("resource_ids") or [] + elif isinstance(decl, list): + ids = decl + else: + ids = [] + return iter(ids) + + minion._iter_resource_ids = _fake_iter minion.gen_modules() minion.connected = True proc_dir = os.path.join(minion_opts["cachedir"], "proc") @@ -4172,6 +4197,30 @@ def test_minion_manager_destroy_after_closed_io_loop_does_not_raise( # -- cancelling a task that never got a single iteration doesn't # exercise the "Event loop is closed" hazard being guarded here. manager.io_loop.run_until_complete(asyncio.sleep(0.2)) + + # Close the event publisher BEFORE closing the loop so its + # PubServer drains its subscribers cleanly (each client's + # ``_stream_read`` task exits via ``StreamClosedError``). The + # per-subscriber tasks are not in ``_background_tasks``, so + # without this they survive loop teardown and later cross a + # test boundary -- on CPython 3.14 asyncio then emits + # ``Task was destroyed but it is pending!`` from a subsequent + # test's caplog, breaking #70748's regression check purely as + # a test-cleanup artifact. See siblings + # ``test_minion_manager_spawn_minions_keeps_task_reference`` + # and ``test_minion_manager_bind_keeps_publisher_task_reference`` + # for the equivalent drain pattern on tracked tasks. Cancel + # the manager's own background tasks the same way and drain + # them before closing the loop. + if manager.event_publisher is not None: + with contextlib.suppress(Exception): + manager.event_publisher.close() + for task in list(manager._background_tasks): + task.cancel() + with contextlib.suppress(Exception): + manager.io_loop.run_until_complete( + asyncio.gather(*manager._background_tasks, return_exceptions=True) + ) manager.io_loop.close() manager.destroy() diff --git a/tests/pytests/unit/test_minion_resources.py b/tests/pytests/unit/test_minion_resources.py index bbb515392e0a..b397d544b380 100644 --- a/tests/pytests/unit/test_minion_resources.py +++ b/tests/pytests/unit/test_minion_resources.py @@ -27,12 +27,79 @@ @pytest.fixture def minion_with_resources(minion_opts): - """A Minion instance with resources configured, no real master connection.""" + """A Minion instance with resources configured, no real master connection. + + The ``opts["resources"]`` assignment is a test-only shim: after the + pillar-to-opts refactor the minion no longer reads a central + ``opts["resources"]`` dict, so we also wire up ``resource_types`` + + patch ``_iter_resource_ids`` to yield the same IDs each test + expects. Tests themselves retain their original semantics. + + The ``_resource_grains_cache`` attribute is no longer an authoritative + accumulator (the register loop writes ``None``; see the memory- + ceiling invariant in ``feedback_minion_never_holds_full_resource_set``), + so tests that preload it get a shim here: we intercept the write and + build a ``dummy.grains`` / ``ssh.grains`` resource_funcs pair that + returns each SRN's grains on demand -- the same shape the real + grains loader would present after per-type streaming. + """ minion_opts["resources"] = RESOURCES + minion_opts["resource_types"] = list(RESOURCES.keys()) minion_opts["multiprocessing"] = False with mock_patch("salt.minion.Minion.gen_modules"): with mock_patch("salt.minion.Minion.connect_master"): m = salt.minion.Minion(minion_opts, load_grains=False) + + # Stream per-type IDs from the static RESOURCES dict so target + # resolution sees the same inventory the pre-refactor tests set up. + def _fake_iter(resource_type): + return iter(RESOURCES.get(resource_type, [])) + + m._iter_resource_ids = _fake_iter + + # Per-type grains callables that honor the resource_ctxvar the real + # loader context sets around ``.grains()`` calls. The test shim + # writes ``_resource_grains_cache = {...}`` via the setter below + # which forwards the mapping into ``_cache_backing`` for this + # lookup. Tests that set the cache directly get grains dispatched + # per-type as the new code does. + cache_backing = {} + m._cache_backing = cache_backing + + def _grains_for(rtype): + def _fn(): + import salt.loader.context as _ctx + + target = _ctx.resource_ctxvar.get() + srn = f"{target['type']}:{target['id']}" + return cache_backing.get(srn, {}) + + return _fn + + m.resource_funcs = { + "dummy.grains": _grains_for("dummy"), + "ssh.grains": _grains_for("ssh"), + } + + # Tests set ``minion._resource_grains_cache = {...}`` as a convenience. + # Intercept the write by swapping the instance's class to a lightweight + # subclass with a property so the per-instance patch doesn't leak to + # every other ``Minion`` built in the test suite. + cls = type(m) + + class _MinionWithCacheShim(cls): + @property + def _resource_grains_cache(self): + return self.__dict__.get("__resource_grains_cache_val") + + @_resource_grains_cache.setter + def _resource_grains_cache(self, value): + cache_backing.clear() + if value: + cache_backing.update(value) + self.__dict__["__resource_grains_cache_val"] = value + + m.__class__ = _MinionWithCacheShim return m @@ -76,11 +143,13 @@ def test_pure_resource_target_excludes_own_minion_id(minion_opts): """A bare id that is BOTH the minion's own id and a managed resource id is not a pure-resource target, so the minion still runs its own job.""" minion_opts["resources"] = RESOURCES + minion_opts["resource_types"] = list(RESOURCES.keys()) minion_opts["multiprocessing"] = False minion_opts["id"] = "dummy-01" with mock_patch("salt.minion.Minion.gen_modules"): with mock_patch("salt.minion.Minion.connect_master"): m = salt.minion.Minion(minion_opts, load_grains=False) + m._iter_resource_ids = lambda rt: iter(RESOURCES.get(rt, [])) own_id = {"tgt": "dummy-01", "tgt_type": "glob", "fun": "test.ping", "arg": []} other_resource = { "tgt": "dummy-02", @@ -367,238 +436,6 @@ def test_thread_return_grain_swap_source_inspection(): assert "minion_instance.resource_funcs[grains_fn]()" in source -# --------------------------------------------------------------------------- -# _discover_resources tests -# --------------------------------------------------------------------------- - - -def test_discover_resources_no_pillar_key_clears_like_empty(minion_with_resources): - """ - When the pillar contains no 'resources' key at all, _discover_resources - must behave like pillar['resources'] == {}: return {} and not preserve - stale opts["resources"]. - """ - minion_with_resources.opts["pillar"] = {} - result = minion_with_resources._discover_resources() - assert result == {} - - -def test_discover_resources_empty_pillar_key_clears_opts(minion_with_resources): - """ - When the pillar *does* contain a 'resources' key but its value is empty, - _discover_resources must return {} and NOT preserve the old opts resources. - This is the fix for the stale-cache bug: removing a resource type from the - pillar and running sync_all must clear it at runtime. - """ - minion_with_resources.opts["pillar"] = {"resources": {}} - result = minion_with_resources._discover_resources() - assert ( - result == {} - ), "_discover_resources must return {} when pillar['resources'] is empty" - - -# --------------------------------------------------------------------------- -# _discover_resources error paths and return-shape contract -# --------------------------------------------------------------------------- - - -def _fake_resource_loader(discovers): - """Return a dict-like loader whose ``.discover`` callables come - from the ``discovers`` mapping ({rtype: callable}). Types absent from - ``discovers`` have no ``.discover`` key — mirrors a loader where - the discover function is missing for that type.""" - return {f"{rtype}.discover": fn for rtype, fn in discovers.items()} - - -def test_discover_resources_continues_when_one_type_raises( - minion_with_resources, caplog -): - """ - A single resource type whose ``discover()`` raises must not block - discovery of the other types. The raising type is omitted from the - result; a warning is logged. - """ - minion_with_resources.opts["pillar"] = { - "resources": {"good_a": {}, "boom": {}, "good_b": {}} - } - fake_loader = _fake_resource_loader( - { - "good_a": lambda opts: ["a-1"], - "boom": lambda opts: (_ for _ in ()).throw( - RuntimeError("connection refused") - ), - "good_b": lambda opts: ["b-1", "b-2"], - } - ) - with mock_patch("salt.loader.resource", return_value=fake_loader): - with caplog.at_level("WARNING"): - result = minion_with_resources._discover_resources() - - assert result == {"good_a": ["a-1"], "good_b": ["b-1", "b-2"]} - assert any( - "Resource discovery failed for type 'boom'" in rec.message - for rec in caplog.records - ), "Expected a warning naming the failing type" - - -def test_discover_resources_skips_type_missing_discover_fn( - minion_with_resources, caplog -): - """ - A type listed in pillar whose loader has no ``.discover`` is - skipped with a warning; other types still discover normally. - """ - minion_with_resources.opts["pillar"] = { - "resources": {"with_disc": {}, "no_disc": {}} - } - fake_loader = _fake_resource_loader( - {"with_disc": lambda opts: ["w-1"]} - ) # "no_disc.discover" intentionally absent - with mock_patch("salt.loader.resource", return_value=fake_loader): - with caplog.at_level("WARNING"): - result = minion_with_resources._discover_resources() - - assert result == {"with_disc": ["w-1"]} - assert any( - "No resource module found for type 'no_disc'" in rec.message - for rec in caplog.records - ), "Expected a warning naming the type missing its discover function" - - -def test_discover_resources_drops_type_when_discover_returns_none( - minion_with_resources, -): - """ - ``discover()`` returning ``None`` is treated as "no ids for this type" - — the type is omitted from the result entirely. Pins the current - ``if ids:`` guard at minion.py:596. - """ - minion_with_resources.opts["pillar"] = {"resources": {"empty": {}, "populated": {}}} - fake_loader = _fake_resource_loader( - { - "empty": lambda opts: None, - "populated": lambda opts: ["p-1"], - } - ) - with mock_patch("salt.loader.resource", return_value=fake_loader): - result = minion_with_resources._discover_resources() - - assert result == {"populated": ["p-1"]} - assert "empty" not in result - - -def test_discover_resources_drops_type_when_discover_returns_empty_list( - minion_with_resources, -): - """ - ``discover()`` returning ``[]`` is treated like ``None`` — type - omitted from result, no empty entry created. - """ - minion_with_resources.opts["pillar"] = {"resources": {"empty": {}}} - fake_loader = _fake_resource_loader({"empty": lambda opts: []}) - with mock_patch("salt.loader.resource", return_value=fake_loader): - result = minion_with_resources._discover_resources() - - assert result == {} - - -def test_discover_resources_coerces_tuple_return_to_list(minion_with_resources): - """ - ``discover()`` returning a tuple is wrapped in ``list()`` (minion.py:597), - so downstream code consistently sees lists. - """ - minion_with_resources.opts["pillar"] = {"resources": {"tup": {}}} - fake_loader = _fake_resource_loader({"tup": lambda opts: ("t-1", "t-2")}) - with mock_patch("salt.loader.resource", return_value=fake_loader): - result = minion_with_resources._discover_resources() - - assert result == {"tup": ["t-1", "t-2"]} - assert isinstance(result["tup"], list) - - -def test_discover_resources_coerces_generator_return_to_list(minion_with_resources): - """ - ``discover()`` returning a generator is fully drained into a list, - matching the tuple-coercion contract. - """ - minion_with_resources.opts["pillar"] = {"resources": {"gen": {}}} - - def gen_discover(opts): - yield "g-1" - yield "g-2" - yield "g-3" - - fake_loader = _fake_resource_loader({"gen": gen_discover}) - with mock_patch("salt.loader.resource", return_value=fake_loader): - result = minion_with_resources._discover_resources() - - assert result == {"gen": ["g-1", "g-2", "g-3"]} - assert isinstance(result["gen"], list) - - -def test_discover_resources_stores_dynamic_ids_verbatim(minion_with_resources): - """ - Pins the current contract: ``discover()`` is the sole source of ids - for its type. The framework does NOT merge with pillar-declared ids. - - Pillar declares ``dynamic_test`` with ``resource_ids: ["p-1", "p-2"]`` - (the values a type like dummy/ssh would read), but ``discover()`` - returns ``["d-1", "d-2"]`` — the result is exactly ``discover()``'s - output, with the pillar ids dropped on the floor. - - This test will need updating when union-with-override lands as a - framework-level merge for the declarative-resources work. - """ - minion_with_resources.opts["pillar"] = { - "resources": { - "dynamic_test": {"resource_ids": ["p-1", "p-2"]}, - } - } - fake_loader = _fake_resource_loader({"dynamic_test": lambda opts: ["d-1", "d-2"]}) - with mock_patch("salt.loader.resource", return_value=fake_loader): - result = minion_with_resources._discover_resources() - - assert result == {"dynamic_test": ["d-1", "d-2"]}, ( - "discover() output is currently authoritative; pillar's " - "resource_ids are NOT merged in by the framework." - ) - - -# --------------------------------------------------------------------------- -# _register_resources_with_master tests -# --------------------------------------------------------------------------- - - -def test_register_resources_with_master_sends_empty_dict(minion_with_resources): - """ - _register_resources_with_master must send the registration even when - opts["resources"] is {}. Without this, removing resources from the pillar - and running sync_all leaves the master cache permanently stale. - """ - minion_with_resources.opts["resources"] = {} - sent_loads = [] - - async def fake_send(load, timeout=None): - sent_loads.append(load) - - import asyncio - - minion_with_resources.tok = b"test-tok" - with mock_patch.object( - minion_with_resources, - "_send_req_async_main", - side_effect=fake_send, - ): - asyncio.run(minion_with_resources._register_resources_with_master()) - - assert ( - len(sent_loads) == 1 - ), "_register_resources_with_master must always send a load, even for {}" - assert ( - sent_loads[0]["resources"] == {} - ), "An empty resource dict must be forwarded to the master to clear stale cache" - - # --------------------------------------------------------------------------- # _MERGE_RESOURCE_FUNS tests # --------------------------------------------------------------------------- @@ -1009,8 +846,12 @@ def test_resolve_resource_targets_grain_lazy_collects_when_cache_missing( ): """ If the grain cache has never been populated (e.g. registration hasn't - happened yet), the resolver falls back to ``_collect_resource_grains`` - so a freshly-started minion still acts on grain targets. + happened yet), the resolver falls back to per-type + ``_collect_resource_grains`` so a freshly-started minion still acts + on grain targets. Post-memory-ceiling refactor: the resolver does + NOT backfill the cache -- grains are rendered on demand per type + and discarded, otherwise a 200K-resource minion would accumulate + the full grain set on first grain-targeted job. """ minion_with_resources._resource_grains_cache = None seen = [] @@ -1029,8 +870,8 @@ def grains_fn(): targets = minion_with_resources._resolve_resource_targets(load) ids = {t["id"] for t in targets} assert ids == {"dummy-01", "dummy-02", "dummy-03"} - # Cache populated on the fly, persisted for the next call. - assert minion_with_resources._resource_grains_cache is not None + # Cache intentionally NOT backfilled -- memory-ceiling invariant. + assert minion_with_resources._resource_grains_cache is None def test_resolve_resource_targets_compound_G_at_grain(minion_with_resources): @@ -1374,3 +1215,119 @@ async def capture(data): reloaded = salt.payload.loads(salt.payload.dumps(delivered[0])) assert "resource_targets" not in reloaded assert reloaded["resource_target"] == delivered[0]["resource_target"] + + +# --------------------------------------------------------------------------- +# Register-loop memory-ceiling regression (Concern 2 of the pre-Phase-3 fix) +# +# The register loop must NOT accumulate per-chunk grain dicts into a +# ``_resource_grains_cache`` attribute: for a 200K-resource inventory +# that would hold ~170 MiB of grain data resident even though the +# chunk shape was supposed to bound memory to one chunk at a time. +# --------------------------------------------------------------------------- + + +async def test_register_loop_does_not_accumulate_grain_cache( + minion_with_resources, +): + """ + After ``_register_resources_with_master`` completes, the minion's + ``_resource_grains_cache`` must be ``None`` -- the chunk's grains + are rendered, shipped, and dropped; nothing accumulates. + + Precedent: Implementer A's first cut rendered grains per chunk but + then wrote every chunk's grains into + ``self._resource_grains_cache``, which re-introduced the + ``total * per-SRN-cost`` memory ceiling violation for 200K- + resource inventories. + """ + + # Capture every chunk the register loop would send. + sent_chunks = [] + + async def _fake_send_req(load, timeout): + sent_chunks.append(dict(load)) + return True + + # Simple per-resource grains callable -- returns the resource ID as + # a grain so each chunk ships a non-empty resource_grains dict. + def _dummy_grains(): + target = salt.loader.context.resource_ctxvar.get() + return {"env": "prod", "resource_id": target["id"]} + + minion_with_resources.resource_funcs = {"dummy.grains": _dummy_grains} + minion_with_resources.tok = b"t" + minion_with_resources._send_req_async_main = _fake_send_req + minion_with_resources._return_retry_timer = lambda: 5 + + # Force a chunk_size of 1 so every ID becomes its own chunk -- a + # multi-chunk register for the 3 dummy IDs in RESOURCES. + minion_with_resources.opts["resource_register_chunk_size"] = 1 + + # Pre-seed the attribute with a non-None sentinel to catch the + # (incorrect) "set to aggregated dict" code path: the loop must + # OVERWRITE it with ``None``, not append. + minion_with_resources._resource_grains_cache = {"stale": "entry"} + + await minion_with_resources._register_resources_with_master() + + # 3 dummy IDs + 1 terminal chunk per type (ssh type sends a final + # empty chunk even though it has no grains). Each chunk's + # resource_grains dict should carry at most that chunk's SRNs. + assert sent_chunks, "register loop must send at least one chunk" + for chunk in sent_chunks: + chunk_ids = chunk.get("resources", {}).get(chunk.get("chunk_rtype"), []) + grain_keys = set(chunk.get("resource_grains", {}).keys()) + assert len(grain_keys) <= max(len(chunk_ids), 0), ( + f"chunk {chunk.get('chunk_rtype')}#{chunk.get('chunk_index')} " + f"shipped {len(grain_keys)} grain dicts for only " + f"{len(chunk_ids)} IDs; grain-cache accumulation across " + "chunks is forbidden" + ) + + # Invariant: no grain-cache accumulator remains on the minion. + assert minion_with_resources._resource_grains_cache is None, ( + "register loop must NOT populate ``_resource_grains_cache`` -- " + "for a 200K inventory this would hold ~170 MiB resident; got " + f"{type(minion_with_resources._resource_grains_cache).__name__}" + ) + + +async def test_register_loop_terminal_chunk_fired_per_type( + minion_with_resources, +): + """ + Every enabled type produces exactly one ``chunk_final=True`` chunk, + and non-final chunks have ``chunk_final=False``. The master-side + stale-SRN prune key depends on exactly-one-terminal-per-type. + """ + sent_chunks = [] + + async def _fake_send_req(load, timeout): + sent_chunks.append(dict(load)) + return True + + minion_with_resources.resource_funcs = {"dummy.grains": lambda: {}} + minion_with_resources.tok = b"t" + minion_with_resources._send_req_async_main = _fake_send_req + minion_with_resources._return_retry_timer = lambda: 5 + # chunk_size=2 against 3 dummy IDs: 2 chunks of (2 ids, 1 id) per + # dummy -- first non-final, second final. + minion_with_resources.opts["resource_register_chunk_size"] = 2 + + await minion_with_resources._register_resources_with_master() + + final_by_type = {} + for chunk in sent_chunks: + rtype = chunk.get("chunk_rtype") + if chunk.get("chunk_final"): + final_by_type.setdefault(rtype, 0) + final_by_type[rtype] += 1 + + # Each enabled type must produce exactly one terminal chunk. + enabled = {"dummy", "ssh"} + for rtype in enabled: + assert final_by_type.get(rtype) == 1, ( + f"expected exactly one chunk_final=True for type {rtype}; " + f"got {final_by_type.get(rtype)}" + ) diff --git a/tests/pytests/unit/utils/test_grain_target_cache.py b/tests/pytests/unit/utils/test_grain_target_cache.py new file mode 100644 index 000000000000..a2a2210acb85 --- /dev/null +++ b/tests/pytests/unit/utils/test_grain_target_cache.py @@ -0,0 +1,399 @@ +""" +Tests for the in-process sorted-key cache that short-circuits +``resource_id:`` predicates with a non-empty fixed prefix. + +Target module: :mod:`salt.utils.grain_target_cache` (Mission 2). The +module provides at minimum: + +* ``get_sorted_keys(cache, bank) -> list[str]`` -- lazily materialised, + sorted list of bank keys; rebuilt when the backing bank changes. +* ``fixed_prefix_of(pattern) -> str | None`` -- returns the characters + of ``pattern`` before the first :mod:`fnmatch` metacharacter + (``*``/``?``/``[``), or ``""``/``None`` when no fixed prefix exists. + +These tests are written from the spec contract; they intentionally +avoid asserting on private implementation details (the specific +binary-search entry point, the shape of any internal LRU, etc.). If +the implementation names differ slightly, adjust the import shim near +the top and the tests will exercise the same behavioural invariants. +""" + +import random +import threading + +import pytest + +# --------------------------------------------------------------------------- +# Import shim — pick the module up if it has landed; skip otherwise. +# --------------------------------------------------------------------------- + +grain_target_cache = pytest.importorskip( + "salt.utils.grain_target_cache", + reason="Mission 2 target module not yet landed", +) + + +def _resolve(*names): + """ + Return the first attribute of :mod:`grain_target_cache` whose name + matches one of ``names``. ``None`` if none is present. + """ + for name in names: + fn = getattr(grain_target_cache, name, None) + if fn is not None: + return fn + return None + + +fixed_prefix_of = _resolve("fixed_prefix_of", "fixed_prefix", "prefix_of_glob") +get_sorted_keys = _resolve("get_sorted_keys", "sorted_keys", "sorted_keys_for_bank") + + +requires_fixed_prefix = pytest.mark.skipif( + fixed_prefix_of is None, reason="fixed_prefix_of not exported by module" +) +requires_get_sorted_keys = pytest.mark.skipif( + get_sorted_keys is None, reason="get_sorted_keys not exported by module" +) + + +# --------------------------------------------------------------------------- +# Fake cache used across tests. Modelled on salt.cache.Cache: the only +# method the sorted-key cache is contractually allowed to call is +# ``list(bank)``; some implementations may also look at a generation +# counter / mtime. +# --------------------------------------------------------------------------- + + +class FakeCache: + """ + Minimal in-memory cache with ``list(bank)`` + mutation helpers. A + ``cachedir`` attribute lets the cache-under-test key its internal + state by (cachedir, bank), matching the real multi-tenant layout. + """ + + def __init__(self, cachedir="/fake/cachedir", banks=None): + self.cachedir = cachedir + self._banks = {bank: list(keys) for bank, keys in (banks or {}).items()} + self.list_calls = 0 + + def list(self, bank): + self.list_calls += 1 + if bank not in self._banks: + raise Exception(f"bank {bank!r} not found") + return list(self._banks[bank]) + + # ---- mutation helpers for tests ---- + def set_bank(self, bank, keys): + self._banks[bank] = list(keys) + + def add(self, bank, key): + self._banks.setdefault(bank, []).append(key) + + def remove(self, bank, key): + self._banks.get(bank, []).remove(key) + + +# --------------------------------------------------------------------------- +# fixed_prefix_of +# --------------------------------------------------------------------------- + + +@requires_fixed_prefix +def test_fixed_prefix_of_trailing_star(): + assert fixed_prefix_of("a-dummy-*") == "a-dummy-" + + +@requires_fixed_prefix +def test_fixed_prefix_of_star_after_literal_digits(): + assert fixed_prefix_of("a-dummy-0010*") == "a-dummy-0010" + + +@requires_fixed_prefix +def test_fixed_prefix_of_pure_star_has_no_prefix(): + # Spec: no fixed prefix -> either empty string or None. + assert fixed_prefix_of("*") in (None, "") + + +@requires_fixed_prefix +def test_fixed_prefix_of_char_class_first_has_no_prefix(): + assert fixed_prefix_of("[abc]foo") in (None, "") + + +@requires_fixed_prefix +def test_fixed_prefix_of_literal_returns_whole_pattern(): + """ + A literal (no metachars) has a prefix equal to the whole pattern. + This is OK: Mission 1 handles the literal fast path separately. + """ + assert fixed_prefix_of("literal") == "literal" + + +@requires_fixed_prefix +def test_fixed_prefix_of_empty_pattern_has_no_prefix(): + assert fixed_prefix_of("") in (None, "") + + +@requires_fixed_prefix +def test_fixed_prefix_of_truncates_at_first_metachar(): + # ``?`` is the first metachar -> prefix is the single leading char. + assert fixed_prefix_of("a?b*c") == "a" + + +@requires_fixed_prefix +def test_fixed_prefix_of_question_mark_only(): + assert fixed_prefix_of("?") in (None, "") + + +@requires_fixed_prefix +def test_fixed_prefix_of_question_mark_after_literal(): + assert fixed_prefix_of("foo?bar") == "foo" + + +@requires_fixed_prefix +def test_fixed_prefix_of_nontrivial_prefix_then_charclass(): + assert fixed_prefix_of("web-[0-9]") == "web-" + + +@requires_fixed_prefix +def test_fixed_prefix_of_backslash_is_not_metachar(): + """ + :mod:`fnmatch` does not treat ``\\`` as an escape; the prefix + extractor must treat ``\\`` as an ordinary character. The exact + output is implementation-defined -- document whatever the + implementation chose so a regression in behaviour is caught. + """ + out = fixed_prefix_of(r"a\*-foo") + assert isinstance(out, (str, type(None))) + # Whatever the choice, it must at least start with ``a`` -- the + # first metachar is ``*`` at index 2 (after ``a\``). + if out: + assert out.startswith("a\\") or out.startswith("a"), out + + +@requires_fixed_prefix +def test_fixed_prefix_of_none_input_safe(): + # Defensive: a None input should not crash the planner. + # Implementation may raise TypeError or return "" / None; accept + # either behaviour but not a crash in production path. + try: + out = fixed_prefix_of(None) + except (TypeError, AttributeError): + return + assert out in (None, "") + + +# --------------------------------------------------------------------------- +# get_sorted_keys — correctness + caching behaviour +# --------------------------------------------------------------------------- + + +@requires_get_sorted_keys +def test_get_sorted_keys_returns_sorted_list(): + cache = FakeCache( + banks={"resource_grains": ["dummy:c", "dummy:a", "dummy:b"]}, + ) + out = get_sorted_keys(cache, "resource_grains") + assert out == sorted(["dummy:c", "dummy:a", "dummy:b"]) + + +@requires_get_sorted_keys +def test_get_sorted_keys_empty_bank_returns_empty_list(): + cache = FakeCache(banks={"resource_grains": []}) + assert get_sorted_keys(cache, "resource_grains") == [] + + +@requires_get_sorted_keys +def test_get_sorted_keys_missing_bank_returns_empty_list(): + """ + A bank the driver refuses / raises on must not blow up the planner; + the sorted-key helper must degrade to an empty list. + """ + cache = FakeCache(banks={}) + out = get_sorted_keys(cache, "no_such_bank") + assert out == [] + + +@requires_get_sorted_keys +def test_get_sorted_keys_caches_without_bank_change(): + """ + Two back-to-back calls over an unchanged bank must hit the cache: + the underlying ``cache.list(bank)`` is called at most a bounded + small number of times (ideally 1; allow up to 2 to account for a + cheap generation/mtime probe). + """ + cache = FakeCache( + banks={"resource_grains": [f"dummy:k-{i:04d}" for i in range(200)]}, + ) + first = get_sorted_keys(cache, "resource_grains") + second = get_sorted_keys(cache, "resource_grains") + assert first == second + # Cache-hit on the second call: at most one additional ``list``. + assert cache.list_calls <= 2, ( + f"get_sorted_keys did not cache -- list() called {cache.list_calls} " + "times for two identical queries" + ) + + +@requires_get_sorted_keys +def test_get_sorted_keys_rebuilds_when_bank_changes(): + cache = FakeCache(banks={"resource_grains": ["dummy:a", "dummy:b"]}) + first = get_sorted_keys(cache, "resource_grains") + assert first == ["dummy:a", "dummy:b"] + + cache.add("resource_grains", "dummy:c") + # The sorted cache must detect the mutation on its next query. + second = get_sorted_keys(cache, "resource_grains") + assert "dummy:c" in second + assert second == sorted(second) + + +@requires_get_sorted_keys +def test_get_sorted_keys_rebuilds_on_shrink(): + cache = FakeCache(banks={"resource_grains": ["dummy:a", "dummy:b", "dummy:c"]}) + assert get_sorted_keys(cache, "resource_grains") == [ + "dummy:a", + "dummy:b", + "dummy:c", + ] + cache.remove("resource_grains", "dummy:b") + assert get_sorted_keys(cache, "resource_grains") == ["dummy:a", "dummy:c"] + + +@requires_get_sorted_keys +def test_get_sorted_keys_multiple_banks_cached_independently(): + """ + Mutating bank A must not evict the cached sort of bank B. + """ + cache = FakeCache( + banks={ + "a": ["a:1", "a:2"], + "b": ["b:1", "b:2"], + }, + ) + assert get_sorted_keys(cache, "a") == ["a:1", "a:2"] + assert get_sorted_keys(cache, "b") == ["b:1", "b:2"] + + before_a = cache.list_calls + _ = get_sorted_keys(cache, "b") + _ = get_sorted_keys(cache, "b") + # Querying ``b`` repeatedly must not re-list ``a`` nor do more + # work than a single list of ``b``. + assert cache.list_calls <= before_a + 2 + + +@requires_get_sorted_keys +def test_get_sorted_keys_separate_cachedirs_are_independent(): + """ + Multi-tenant masters: one cachedir's sorted view must not shadow + another's. The spec names ``(cache_dir, bank_name)`` as the cache + key. + """ + cache1 = FakeCache(cachedir="/c1", banks={"resource_grains": ["dummy:a"]}) + cache2 = FakeCache(cachedir="/c2", banks={"resource_grains": ["dummy:z"]}) + + assert get_sorted_keys(cache1, "resource_grains") == ["dummy:a"] + assert get_sorted_keys(cache2, "resource_grains") == ["dummy:z"] + # Asking ``cache1`` again must still return ``cache1``'s view, not + # the last-seen-any-cache view. + assert get_sorted_keys(cache1, "resource_grains") == ["dummy:a"] + + +@requires_get_sorted_keys +def test_get_sorted_keys_thread_safety_concurrent_reads(): + """ + Hammer the sorted-key helper from many threads against a stable + bank. The result must always be consistent (sorted, deduplicated, + correct length) and the helper must never raise. + """ + keys = [f"dummy:k-{i:05d}" for i in range(2000)] + random.Random(1234).shuffle(keys) + cache = FakeCache(banks={"resource_grains": keys}) + expected = sorted(keys) + + errors = [] + results = [] + lock = threading.Lock() + barrier = threading.Barrier(16) + + def worker(): + try: + barrier.wait(timeout=30) + for _ in range(20): + out = get_sorted_keys(cache, "resource_grains") + # Validate shape eagerly inside the thread so a mid-test + # mutation is surfaced as a race rather than a stale + # read. + if list(out) != expected: + with lock: + results.append(out) + break + except Exception as exc: # pylint: disable=broad-except + # pragma: no cover - visible via errors + with lock: + errors.append(exc) + + threads = [threading.Thread(target=worker) for _ in range(16)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=60) + assert not errors, errors + assert not results, f"stale/inconsistent reads from worker threads: {results[:2]}" + + +# --------------------------------------------------------------------------- +# Invariant: fixed_prefix_of output is a strict prefix of every +# fnmatch-matching string. This is the guarantee that lets a binary +# search over sorted keys bound the scan. +# --------------------------------------------------------------------------- + + +@requires_fixed_prefix +@pytest.mark.parametrize( + "pattern", + [ + "a-*", + "web-*", + "a-dummy-0010*", + "foo?bar", + "x[abc]", + "literal", + "a?b*c", + "web-[0-9]", + ], +) +def test_fixed_prefix_is_prefix_of_any_fnmatch_match(pattern): + """ + For every pattern, any string that ``fnmatchcase`` matches must + start with ``fixed_prefix_of(pattern)``. This is the core + correctness invariant that justifies the binary-search range. + """ + import fnmatch as _fnmatch + + prefix = fixed_prefix_of(pattern) + if not prefix: + return # trivially holds for empty prefix + + sample_universe = [ + "a", + "a-", + "a-dummy-", + "a-dummy-0010", + "a-dummy-00100", + "web-01", + "web-foo", + "foo-bar", + "fooxbar", + "literal", + "xa", + "xaabc", + "aXbYc", + ] + for s in sample_universe: + if _fnmatch.fnmatchcase(s, pattern): + assert s.startswith(prefix), ( + f"pattern={pattern!r} prefix={prefix!r} matched string " + f"{s!r} that does not start with the prefix -- binary " + f"search would miss it" + ) diff --git a/tests/pytests/unit/utils/test_minions_resources.py b/tests/pytests/unit/utils/test_minions_resources.py index 4bd2e9f54555..3ffc31478dae 100644 --- a/tests/pytests/unit/utils/test_minions_resources.py +++ b/tests/pytests/unit/utils/test_minions_resources.py @@ -505,25 +505,6 @@ def test_check_minions_glob_exact_bare_registered_resource_id(ck_with_resources) assert "dummy-02" in result["minions"] -def test_check_glob_minions_bare_resource_from_pillar_cache_empty_registry(opts): - """Bare glob matches cached pillar ``resources`` when mmap registry is cold.""" - opts["minion_data_cache"] = True - fake_cache = MagicMock() - - def fetch_side_effect(bank, mid): - if bank == "pillar" and mid == "m2": - return {"resources": {"dummy": {"resource_ids": ["m2-dummy2"]}}} - return {} - - fake_cache.list.side_effect = lambda bank: ["m2"] - fake_cache.fetch.side_effect = fetch_side_effect - - with patch("salt.cache.factory", return_value=fake_cache): - ck = salt.utils.minions.CkMinions(opts) - result = ck._check_glob_minions("m2-dummy2", greedy=True) - assert "m2-dummy2" in result["minions"] - - def test_check_glob_minions_bare_resource_from_grains_cache_empty_registry(opts): opts["minion_data_cache"] = True fake_cache = MagicMock() @@ -542,25 +523,6 @@ def fetch_side_effect(bank, mid): assert "m2-dummy2" in result["minions"] -def test_check_list_minions_bare_resource_from_cache_empty_registry(opts): - opts["minion_data_cache"] = True - fake_cache = MagicMock() - fake_cache.list.side_effect = lambda bank: ["m2"] - - def fetch_side_effect(bank, mid): - if bank == "pillar" and mid == "m2": - return {"resources": {"dummy": {"resource_ids": ["m2-dummy2"]}}} - return {} - - fake_cache.fetch.side_effect = fetch_side_effect - - with patch("salt.cache.factory", return_value=fake_cache): - ck = salt.utils.minions.CkMinions(opts) - result = ck._check_list_minions("m2-dummy2", greedy=True) - assert result["minions"] == ["m2-dummy2"] - assert result["missing"] == [] - - # --------------------------------------------------------------------------- # Bare resource ID targeting — master path + concurrency # --------------------------------------------------------------------------- @@ -749,6 +711,341 @@ def test_augment_grain_match_invalid_srn_skipped(opts): assert result["minions"] == ["valid"] +# --------------------------------------------------------------------------- +# _augment_grain_match_with_resource_grains — literal resource_id fast path +# (exact-key hash probe: ``self.cache.contains(bank, ":")``) +# --------------------------------------------------------------------------- + + +def _build_ck_with_contains(opts, present_keys, list_result=None): + """ + Build a ``CkMinions`` whose ``self.cache`` answers: + + * ``contains(bank, key)`` → ``key in present_keys`` for the + ``resource_grains`` bank; ``False`` for everything else. + * ``list(bank)`` → ``list_result`` (defaults to the keys in + ``present_keys``) so the fallback slow path works if the + literal fast path is NOT taken. + + Returns ``(ck, fake_cache)`` so tests can introspect call counts. + """ + ck = salt.utils.minions.CkMinions(opts) + fake = MagicMock() + keys = list(present_keys) + + def _contains(bank, key): + if bank != "resource_grains": + return False + return key in present_keys + + def _list(bank): + if bank != "resource_grains": + return [] + return list(keys if list_result is None else list_result) + + def _fetch(bank, key): + if bank != "resource_grains": + return None + gd = present_keys.get(key) if isinstance(present_keys, dict) else None + return gd + + fake.contains = MagicMock(side_effect=_contains) + fake.list = MagicMock(side_effect=_list) + fake.fetch = MagicMock(side_effect=_fetch) + ck.cache = fake + return ck, fake + + +def test_augment_grain_literal_resource_id_hits_via_contains_probe(opts): + """ + Literal ``resource_id:`` with a single configured type must probe + exactly ``cache.contains("resource_grains", ":")`` and + return the bare id. Crucially, ``cache.list`` must NOT be called -- the + probe is authoritative and avoids the O(bank-size) scan. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + present = {"dummy:dummy-01": {"resource_id": "dummy-01"}} + ck, fake = _build_ck_with_contains(opts, present) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:dummy-01", ":", regex_match=False + ) + + assert result["minions"] == ["dummy-01"] + # Exactly one probe (T == 1); no bank list / fetch on the fast path. + assert fake.contains.call_count == 1 + fake.contains.assert_called_with("resource_grains", "dummy:dummy-01") + assert fake.list.call_count == 0 + assert fake.fetch.call_count == 0 + + +def test_augment_grain_literal_resource_id_miss_probes_every_type(opts): + """ + Literal predicate against an id that is not registered under any + configured type returns no match -- and we probe each configured type + exactly once (``T`` probes total, no fall-through to the scan). + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy", "ssh", "aws"] + ck, fake = _build_ck_with_contains(opts, present_keys={}) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:nosuch", ":", regex_match=False + ) + + assert result["minions"] == [] + assert fake.contains.call_count == 3 + probed = [c.args[1] for c in fake.contains.call_args_list] + assert set(probed) == {"dummy:nosuch", "ssh:nosuch", "aws:nosuch"} + # No fall-through to the scan: the probe is authoritative. + assert fake.list.call_count == 0 + assert fake.fetch.call_count == 0 + + +def test_augment_grain_literal_resource_id_matches_second_type(opts): + """ + Multiple configured types; id is registered under exactly one. The + probe short-circuits on the first hit -- we may make up to T probes + in the worst case, and the matching id is appended once. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy", "ssh"] + # Only registered under ssh, not dummy. + present = {"ssh:node-42": {"resource_id": "node-42"}} + ck, fake = _build_ck_with_contains(opts, present) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:node-42", ":", regex_match=False + ) + + assert result["minions"] == ["node-42"] + # Probe order is enabled_resource_types order: dummy (miss) then ssh (hit). + probed = [c.args[1] for c in fake.contains.call_args_list] + assert probed == ["dummy:node-42", "ssh:node-42"] + assert fake.list.call_count == 0 + + +def test_augment_grain_glob_resource_id_falls_through_to_scan_fast_path(opts): + """ + ``resource_id:a-dummy-*`` has a wildcard, so the literal fast path + must fall through and the existing bank-scan fast path (via + :func:`_resource_id_candidates`) handles it instead. No + ``cache.contains`` calls expected. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + present = { + "dummy:a-dummy-01": {"resource_id": "a-dummy-01"}, + "dummy:a-dummy-02": {"resource_id": "a-dummy-02"}, + "dummy:other": {"resource_id": "other"}, + } + ck, fake = _build_ck_with_contains(opts, present) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:a-dummy-*", ":", regex_match=False + ) + + assert sorted(result["minions"]) == ["a-dummy-01", "a-dummy-02"] + # Fast path for globs is bank-list-based, not contains-probe-based. + assert fake.contains.call_count == 0 + assert fake.list.call_count >= 1 + + +def test_augment_grain_nested_resource_id_predicate_falls_through(opts): + """ + ``resource_id:a:b`` is a nested predicate -- the fast path must NOT + swallow it. Falls through to the slow path (which rejects it via + :func:`_resource_id_candidates` as well, then does the full scan). + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + present = {"dummy:a": {"resource_id": "a"}} + ck, fake = _build_ck_with_contains(opts, present) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:a:b", ":", regex_match=False + ) + + # Nested predicates never hit the hash-probe fast path. + assert fake.contains.call_count == 0 + # No match either way: "resource_id" value in the grain dict is "a", + # which does not nest-match "resource_id:a:b". + assert result["minions"] == [] + + +def test_augment_grain_non_resource_id_predicate_falls_through(opts): + """ + A grain target with a different key (e.g. ``role:web``) must not + trigger the literal-probe fast path. It falls through to the + existing slow path (bank list + per-key fetch + subdict_match). + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + present = { + "dummy:d-1": {"role": "web", "resource_id": "d-1"}, + "dummy:d-2": {"role": "db", "resource_id": "d-2"}, + } + ck, fake = _build_ck_with_contains(opts, present) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "role:web", ":", regex_match=False + ) + + assert result["minions"] == ["d-1"] + assert fake.contains.call_count == 0 + # Slow path: list + fetch per key. + assert fake.list.call_count >= 1 + assert fake.fetch.call_count >= 1 + + +def test_augment_grain_empty_resource_id_predicate_falls_through(opts): + """ + ``resource_id:`` with an empty pattern must not take the fast path + (empty literal would probe ``:`` which is never a valid + bank key). Falls through to the existing behaviour. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + present = {"dummy:x": {"resource_id": "x"}} + ck, fake = _build_ck_with_contains(opts, present) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:", ":", regex_match=False + ) + + assert fake.contains.call_count == 0 + # No bare match for the empty pattern. + assert result["minions"] == [] + + +def test_augment_grain_literal_regex_predicate_does_not_take_fast_path(opts): + """ + PCRE form (``salt -P resource_id:^d``) must bypass the literal-exact + probe -- regex match semantics are not identity-equal to the literal + bank-key test. Falls through to the existing regex-aware scan. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + present = { + "dummy:d-1": {"resource_id": "d-1"}, + "dummy:x-1": {"resource_id": "x-1"}, + } + ck, fake = _build_ck_with_contains(opts, present) + + result = {"minions": [], "missing": []} + # "d-1" is a valid regex (treated as literal by re.search anyway) but + # the fast path MUST NOT trigger when ``regex_match=True``. + ck._augment_grain_match_with_resource_grains( + result, "resource_id:d-1", ":", regex_match=True + ) + + assert fake.contains.call_count == 0 + assert result["minions"] == ["d-1"] + + +def test_augment_grain_literal_no_configured_types_returns_empty(opts): + """ + Edge case: ``opts["resource_types"]`` missing / empty → the fast path + makes zero probes and returns no match. No fall-through to the scan. + """ + opts["minion_data_cache"] = True + opts.pop("resource_types", None) + present = {"dummy:dummy-01": {"resource_id": "dummy-01"}} + ck, fake = _build_ck_with_contains(opts, present) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:dummy-01", ":", regex_match=False + ) + + assert result["minions"] == [] + assert fake.contains.call_count == 0 + # Probe path is authoritative -- the scan must NOT run as a backstop. + assert fake.list.call_count == 0 + + +def test_augment_grain_literal_contains_raises_is_swallowed(opts): + """ + If the cache driver raises on ``contains`` for one type, we continue + to the next type rather than crashing the resolver. A subsequent hit + still returns the id. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy", "ssh"] + ck = salt.utils.minions.CkMinions(opts) + fake = MagicMock() + + def _contains(bank, key): + if key == "dummy:x": + raise RuntimeError("driver glitch") + return key == "ssh:x" + + fake.contains = MagicMock(side_effect=_contains) + fake.list = MagicMock(return_value=[]) + fake.fetch = MagicMock(return_value=None) + ck.cache = fake + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:x", ":", regex_match=False + ) + + assert result["minions"] == ["x"] + assert fake.contains.call_count == 2 + + +def test_augment_grain_literal_enabled_types_raises_is_swallowed(opts): + """ + A failure in ``enabled_resource_types`` must not propagate -- the + fast path treats it as "zero configured types" and returns empty. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + present = {"dummy:dummy-01": {"resource_id": "dummy-01"}} + ck, fake = _build_ck_with_contains(opts, present) + + result = {"minions": [], "missing": []} + with patch( + "salt.utils.resources.enabled_resource_types", + side_effect=RuntimeError("opts corrupt"), + ): + ck._augment_grain_match_with_resource_grains( + result, "resource_id:dummy-01", ":", regex_match=False + ) + + assert result["minions"] == [] + assert fake.contains.call_count == 0 + + +def test_is_literal_glob_helper_shapes(): + """Sanity check for the exported shape helpers.""" + from salt.utils.minions import _is_literal_glob, _is_resource_id_literal_predicate + + assert _is_literal_glob("abc") is True + assert _is_literal_glob("a-dummy-0000100") is True + assert _is_literal_glob("a*") is False + assert _is_literal_glob("a?") is False + assert _is_literal_glob("a[0-9]") is False + assert _is_literal_glob("") is False + assert _is_literal_glob(None) is False + + assert _is_resource_id_literal_predicate("resource_id:dummy-01", ":") is True + assert _is_resource_id_literal_predicate("resource_id:a-*", ":") is False + assert _is_resource_id_literal_predicate("resource_id:a:b", ":") is False + assert _is_resource_id_literal_predicate("resource_id:", ":") is False + assert _is_resource_id_literal_predicate("role:web", ":") is False + assert _is_resource_id_literal_predicate("", ":") is False + assert _is_resource_id_literal_predicate(None, ":") is False + + # --------------------------------------------------------------------------- # Nodegroup expansion with G@ for resources # --------------------------------------------------------------------------- @@ -978,3 +1275,695 @@ def test_check_minions_grain_target_10000_minions_100_resources_each(opts): slowdowns as "timeout" failures. """ _run_check_minions_grain_perf(opts, 10000, 100, budget=180.0) + + +# --------------------------------------------------------------------------- +# Mission 2: sorted-key cache for ``resource_id:`` with a fixed prefix +# +# The query planner must route ``resource_id:*`` style predicates +# through a sorted-key cache that bounds the scan via binary search on the +# fixed prefix. Behavioural contract tests -- no reliance on the internal +# planner entry point beyond the ``_augment_grain_match_with_resource_grains`` +# public surface. +# --------------------------------------------------------------------------- + + +_gtc = pytest.importorskip( + "salt.utils.grain_target_cache", + reason="Mission 2 sorted-key cache module not yet landed", +) + + +def _build_ck_with_keys(opts, keys): + """ + Build a ``CkMinions`` whose ``self.cache`` returns ``keys`` from + ``list("resource_grains")`` and nothing from other banks. Grain + dicts for each key are synthesised on the fly so a slow-path + fallback still produces correct matches (the ``resource_id`` grain + is derived from the SRN). + + Returns ``(ck, fake_cache)`` so tests can inspect the mock. + """ + ck = salt.utils.minions.CkMinions(opts) + fake = MagicMock() + present = list(keys) + + def _list(bank): + return list(present) if bank == "resource_grains" else [] + + def _fetch(bank, key): + if bank != "resource_grains": + return None + _rtype, _, rid = key.partition(":") + if not rid: + return None + return {"resource_id": rid} + + def _contains(bank, key): + if bank != "resource_grains": + return False + return key in present + + fake.list = MagicMock(side_effect=_list) + fake.fetch = MagicMock(side_effect=_fetch) + fake.contains = MagicMock(side_effect=_contains) + ck.cache = fake + return ck, fake + + +def _rids(result): + return sorted(result["minions"]) + + +def test_augment_grain_resource_id_fixed_prefix_matches_only_prefix_keys(opts): + """ + ``resource_id:a-*`` against a bank with ``a-foo``, ``a-bar``, + ``b-baz`` must return exactly ``a-foo`` and ``a-bar``. This is the + behavioural contract regardless of whether the implementation used + the sorted-key fast path or fell through to the existing scan. + """ + opts["minion_data_cache"] = True + ck, _fake = _build_ck_with_keys(opts, ["dummy:a-foo", "dummy:a-bar", "dummy:b-baz"]) + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:a-*", ":", regex_match=False + ) + assert _rids(result) == ["a-bar", "a-foo"] + + +def test_augment_grain_resource_id_fixed_prefix_bounds_fnmatch_calls(opts, tmp_path): + """ + With a sorted-key cache on a 10K-key bank, a ``resource_id:a-*`` + query must not run ``fnmatch`` against every key. We bound the + call count to roughly (matches + a small number of binary-search + probes). A linear scan would hit all 10,000 keys. + """ + import fnmatch + + import salt.utils.grain_target_cache as _gtc_mod + + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + bank_dir = tmp_path / "resource_grains" + bank_dir.mkdir() + cachedir = str(tmp_path) + + # 10,000 keys; prefix ``a-`` matches 100 of them (indices 0..99). + keys = [f"dummy:a-{i:03d}" for i in range(100)] + keys += [f"dummy:b-{i:04d}" for i in range(9900)] + ck, fake = _build_ck_with_keys(opts, keys) + fake.cachedir = cachedir + _gtc_mod.invalidate() + + real_fnmatchcase = fnmatch.fnmatchcase + call_count = {"n": 0} + + def _count(name, pat): + call_count["n"] += 1 + return real_fnmatchcase(name, pat) + + result = {"minions": [], "missing": []} + with patch("fnmatch.fnmatchcase", side_effect=_count): + ck._augment_grain_match_with_resource_grains( + result, "resource_id:a-*", ":", regex_match=False + ) + + assert len(result["minions"]) == 100 + # Sorted-cache path should keep fnmatch calls bounded by + # (matches + a few bsearch probes). Pin an upper bound well below + # the full bank size. On a pure linear scan this would be 10000. + bound = 100 * 2 + 64 + assert call_count["n"] <= bound, ( + f"fnmatchcase called {call_count['n']} times on a 10K bank with " + f"100 matches -- expected <= {bound} with sorted-cache routing" + ) + + +def test_augment_grain_resource_id_no_fixed_prefix_matches_correctly(opts): + """ + ``resource_id:*-foo`` has no fixed prefix. The sorted-cache path + does not apply; we fall through to the existing linear scan but + the match set must still be correct. + """ + opts["minion_data_cache"] = True + ck, _fake = _build_ck_with_keys(opts, ["dummy:a-foo", "dummy:b-foo", "dummy:a-bar"]) + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:*-foo", ":", regex_match=False + ) + assert _rids(result) == ["a-foo", "b-foo"] + + +def test_augment_grain_resource_id_charclass_first_matches_correctly(opts): + """ + ``resource_id:[ab]-*`` leads with a char class -- there is no + fixed prefix. The implementation may fall through to a linear + scan or expand the char class; either way the result set must + match exactly the keys that fnmatch accepts. + """ + opts["minion_data_cache"] = True + ck, _fake = _build_ck_with_keys(opts, ["dummy:a-1", "dummy:b-1", "dummy:c-1"]) + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:[ab]-*", ":", regex_match=False + ) + assert _rids(result) == ["a-1", "b-1"] + + +def test_augment_grain_resource_id_pure_literal_uses_mission1_hash_probe(opts): + """ + A literal predicate (``resource_id:a-dummy-0000100``) is handled + by Mission 1's ``cache.contains`` hash-probe fast path. The + Mission 2 sorted-cache path MUST NOT be consulted. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + present_keys = {"dummy:a-dummy-0000100": {"resource_id": "a-dummy-0000100"}} + + ck = salt.utils.minions.CkMinions(opts) + fake = MagicMock() + fake.contains = MagicMock( + side_effect=lambda b, k: b == "resource_grains" and k in present_keys + ) + fake.list = MagicMock(return_value=[]) + fake.fetch = MagicMock(return_value=None) + ck.cache = fake + + gsk_candidates = ["get_sorted_keys", "sorted_keys", "sorted_keys_for_bank"] + gsk_name = next( + (n for n in gsk_candidates if hasattr(_gtc, n)), + None, + ) + if gsk_name is None: + pytest.skip("grain_target_cache has no recognised sorted-keys entry point") + + with patch.object(_gtc, gsk_name) as sorted_cache_entry: + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, + "resource_id:a-dummy-0000100", + ":", + regex_match=False, + ) + + assert result["minions"] == ["a-dummy-0000100"] + assert sorted_cache_entry.call_count == 0, ( + "literal resource_id predicate should be resolved by the " + "Mission 1 hash-probe path, not the sorted-key cache" + ) + + +def test_augment_grain_non_resource_id_predicate_bypasses_sorted_cache(opts): + """ + ``role:web`` is not a ``resource_id`` predicate -- the sorted-key + cache must not be consulted. The slow pickle-fetch path runs and + the match set is still correct. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + ck = salt.utils.minions.CkMinions(opts) + fake = MagicMock() + entries = { + "dummy:d-1": {"role": "web", "resource_id": "d-1"}, + "dummy:d-2": {"role": "db", "resource_id": "d-2"}, + } + fake.list = MagicMock( + side_effect=lambda b: list(entries) if b == "resource_grains" else [] + ) + fake.fetch = MagicMock( + side_effect=lambda b, k: entries.get(k) if b == "resource_grains" else None + ) + fake.contains = MagicMock(return_value=False) + ck.cache = fake + + gsk_candidates = ["get_sorted_keys", "sorted_keys", "sorted_keys_for_bank"] + gsk_name = next((n for n in gsk_candidates if hasattr(_gtc, n)), None) + if gsk_name is None: + pytest.skip("grain_target_cache has no recognised sorted-keys entry point") + + with patch.object(_gtc, gsk_name) as sorted_cache_entry: + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "role:web", ":", regex_match=False + ) + + assert result["minions"] == ["d-1"] + assert sorted_cache_entry.call_count == 0 + + +def test_augment_grain_resource_id_empty_predicate_does_not_crash(opts): + opts["minion_data_cache"] = True + ck, _fake = _build_ck_with_keys(opts, ["dummy:x"]) + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:", ":", regex_match=False + ) + assert result["minions"] == [] + + +def test_augment_grain_resource_id_pcre_bypasses_sorted_cache(opts): + """ + PCRE form bypasses the fixed-prefix sorted-cache path. The sorted + cache sees nothing; the existing regex scan produces the match. + """ + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + present = { + "dummy:d-1": {"resource_id": "d-1"}, + "dummy:x-1": {"resource_id": "x-1"}, + } + ck = salt.utils.minions.CkMinions(opts) + fake = MagicMock() + fake.list = MagicMock( + side_effect=lambda b: list(present) if b == "resource_grains" else [] + ) + fake.fetch = MagicMock( + side_effect=lambda b, k: present.get(k) if b == "resource_grains" else None + ) + fake.contains = MagicMock(return_value=False) + ck.cache = fake + + gsk_candidates = ["get_sorted_keys", "sorted_keys", "sorted_keys_for_bank"] + gsk_name = next((n for n in gsk_candidates if hasattr(_gtc, n)), None) + + result = {"minions": [], "missing": []} + if gsk_name is None: + ck._augment_grain_match_with_resource_grains( + result, "resource_id:^d-", ":", regex_match=True + ) + else: + with patch.object(_gtc, gsk_name) as sorted_cache_entry: + ck._augment_grain_match_with_resource_grains( + result, "resource_id:^d-", ":", regex_match=True + ) + assert sorted_cache_entry.call_count == 0 + + # The current PCRE path matches ``d-1`` through subdict_match. + assert "d-1" in result["minions"] + assert "x-1" not in result["minions"] + + +def test_augment_grain_resource_id_fixed_prefix_empty_bank_returns_empty(opts): + opts["minion_data_cache"] = True + ck, _fake = _build_ck_with_keys(opts, []) + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:a-*", ":", regex_match=False + ) + assert result["minions"] == [] + + +def test_augment_grain_resource_id_fixed_prefix_cached_across_calls(opts, tmp_path): + """ + Two identical ``resource_id:*`` queries in a row: the + second query must not re-sort the bank. We assert this by + counting ``cache.list`` calls -- a correctly-caching planner hits + ``list`` at most ~1 time per query (one to confirm the generation + hasn't changed; the actual sort is cached). + + The sorted-key cache uses a cheap O(1) bank-generation signal + (mmap occupied-count or bank-directory mtime) to decide whether + to rebuild. We stage a real on-disk bank directory so the + directory-mtime fallback engages. + """ + import os + + import salt.utils.grain_target_cache as _gtc_mod + + opts["minion_data_cache"] = True + opts["resource_types"] = ["dummy"] + bank_dir = tmp_path / "resource_grains" + bank_dir.mkdir() + cachedir = str(tmp_path) + + ck, fake = _build_ck_with_keys(opts, ["dummy:a-1", "dummy:a-2", "dummy:b-1"]) + fake.cachedir = cachedir + # Force a clean cache slate so prior tests sharing the singleton + # don't pollute this assertion. + _gtc_mod.invalidate() + + result1 = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result1, "resource_id:a-*", ":", regex_match=False + ) + first_list_calls = fake.list.call_count + + # Pin the bank-dir mtime so the generation signal does not advance + # between the two queries (otherwise filesystem jitter would force + # a rebuild and this test would be a flake rather than a cache + # invariant check). + st = os.stat(bank_dir) + os.utime(bank_dir, ns=(st.st_atime_ns, st.st_mtime_ns)) + + result2 = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result2, "resource_id:a-*", ":", regex_match=False + ) + second_list_calls = fake.list.call_count + + assert _rids(result1) == _rids(result2) == ["a-1", "a-2"] + # The second query should not materially increase the number of + # list calls -- the sorted view is cached. + assert second_list_calls - first_list_calls <= 1, ( + f"sorted-cache did not short-circuit the second identical query: " + f"list() went from {first_list_calls} to {second_list_calls}" + ) + + +def test_augment_grain_resource_id_dedup_across_calls(opts): + """ + A predicate matching 100 keys, called twice against the same + ``result`` object, must contain each rid exactly once. + """ + opts["minion_data_cache"] = True + keys = [f"dummy:a-{i:03d}" for i in range(100)] + keys += [f"dummy:b-{i:03d}" for i in range(50)] + ck, _fake = _build_ck_with_keys(opts, keys) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:a-*", ":", regex_match=False + ) + assert len(result["minions"]) == 100 + + ck._augment_grain_match_with_resource_grains( + result, "resource_id:a-*", ":", regex_match=False + ) + # No duplicates after the second identical query. + assert len(result["minions"]) == len(set(result["minions"])) == 100 + + +def test_augment_grain_resource_id_fixed_prefix_dedup_on_duplicate_keys(opts): + """ + If the bank somehow returns a duplicate key for the sorted view, + the resulting rid list must still contain each id exactly once. + """ + opts["minion_data_cache"] = True + # ``_build_ck_with_keys`` preserves duplicates as-is. + ck, _fake = _build_ck_with_keys(opts, ["dummy:a-1", "dummy:a-1", "dummy:a-2"]) + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:a-*", ":", regex_match=False + ) + assert sorted(result["minions"]) == ["a-1", "a-2"] + + +@pytest.mark.parametrize( + "pattern", + [ + "a-*", + "a-dummy-001*", + "web-*", + "ns-*", + "foo?-*", + "literal-key-123", + ], +) +def test_augment_grain_resource_id_matches_linear_reference(opts, pattern): + """ + Match-set equivalence: for any glob with a fixed prefix, the + planner's output MUST equal the reference linear-scan match set + over the bank's id portion. Covers a handful of representative + shapes. + """ + import fnmatch as _fnmatch + import random as _random + + opts["minion_data_cache"] = True + # Ensure Mission 1's literal hash-probe can resolve ":" + # candidates; without this, a literal-only pattern returns empty + # because the probe loop iterates zero configured types. + opts["resource_types"] = ["dummy"] + _random.seed(0xC0FFEE) + keys = sorted( + { + f"dummy:{prefix}-{i:04d}" + for prefix in ("a", "a-dummy-001", "a-dummy-002", "web", "ns", "foo1") + for i in range(50) + } + ) + keys.append("dummy:literal-key-123") + _random.shuffle(keys) + + ck, _fake = _build_ck_with_keys(opts, keys) + + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, f"resource_id:{pattern}", ":", regex_match=False + ) + got = sorted(result["minions"]) + + expected = sorted( + { + key.partition(":")[2] + for key in keys + if _fnmatch.fnmatchcase(key.partition(":")[2], pattern) + } + ) + assert got == expected, ( + f"pattern={pattern!r} planner returned {got!r} but linear " + f"reference expects {expected!r}" + ) + + +def test_augment_grain_resource_id_prefix_with_colon_treats_id_only(opts): + """ + Resource IDs are allowed to contain colons; the sorted keys are + stored as ``:`` so a ``resource_id:ns:res-*`` query + carries the first colon INSIDE the id portion. The current + planner already treats a nested predicate specially (via + ``_resource_id_candidates``, which returns ``None`` and falls + back). The Mission 2 sorted-cache path is NOT expected to handle + nested patterns either -- but it must still produce the correct + result, even if that means the slow-path fallback runs. + """ + opts["minion_data_cache"] = True + # One matching key where the id literally contains a colon. + keys = ["dummy:ns:res-1", "dummy:ns:res-2", "dummy:other"] + ck, _fake = _build_ck_with_keys(opts, keys) + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, "resource_id:ns:res-*", ":", regex_match=False + ) + # The nested-predicate rule short-circuits to the slow path. + # subdict_match on {"resource_id": "ns:res-1"} with + # ``resource_id:ns:res-*`` traverses one level and then runs an + # fnmatch-style check on the remainder -- documented as a + # fall-through here, not a strict shape test. Confirm no crash + # and the result is a subset of the matching keys. + for rid in result["minions"]: + assert rid.startswith("ns:res-"), rid + + +def test_augment_grain_resource_id_backslash_in_pattern(opts): + """ + ``resource_id:a\\*-foo`` -- backslash is NOT a glob escape in + :mod:`fnmatch`; the pattern matches a literal ``a\\`` followed by + any chars ending in ``-foo``. The sorted-cache planner must not + misparse the backslash as an escape and silently drop the match. + """ + import fnmatch as _fnmatch + + opts["minion_data_cache"] = True + keys = [r"dummy:a\x-foo", r"dummy:a\y-foo", "dummy:aZ-foo"] + ck, _fake = _build_ck_with_keys(opts, keys) + + pattern = r"a\*-foo" + result = {"minions": [], "missing": []} + ck._augment_grain_match_with_resource_grains( + result, f"resource_id:{pattern}", ":", regex_match=False + ) + expected = sorted( + { + key.partition(":")[2] + for key in keys + if _fnmatch.fnmatchcase(key.partition(":")[2], pattern) + } + ) + assert sorted(result["minions"]) == expected + + +# --------------------------------------------------------------------------- +# Trie dispatch on the per-minion ``grains`` bank. +# +# The trie infrastructure is bank-agnostic. When +# ``opts["enable_grain_trie_index"]`` is set, ``_check_cache_minions`` +# consults :func:`salt.utils.mmap_trie_index.dispatch_predicate` for the +# ``grains`` bank BEFORE the legacy per-minion pickle-fetch loop, so +# arbitrary grain-target predicates on real minions (``os:Linux``, +# ``cpuinfo:vendor_id:AMD``) short-circuit when the trie can +# authoritatively serve them. +# +# These tests pin the contract from the public +# :meth:`_check_grain_minions` surface: +# +# * dispatch is gated on ``enable_grain_trie_index`` -- off by default, +# legacy slow path runs +# * a trie hit skips the ``cache.fetch(bank, id)`` loop entirely +# * trie ``None`` (cap sentinel, missing build) falls through to the +# legacy path so no match is lost +# * trie results intersect with the starting minion set (PKI for +# greedy, cached list for non-greedy) so an accepted-key filter +# isn't bypassed by stale cached grains +# --------------------------------------------------------------------------- + + +def _build_ck_with_minion_grains(opts, minion_grains): + """ + Build a ``CkMinions`` whose ``self.cache`` returns the given + ``{minion_id: {grain_key: grain_value, ...}}`` map from the + ``grains`` bank. The fixture also makes ``self._pki_minions()`` + return every minion id in the map so greedy dispatch includes all + of them as candidates. + """ + ck = salt.utils.minions.CkMinions(opts) + present = dict(minion_grains) + + def _list(bank): + if bank == "grains": + return list(present) + return [] + + def _fetch(bank, key): + if bank != "grains": + return None + return present.get(key) + + fake = MagicMock() + fake.list = MagicMock(side_effect=_list) + fake.fetch = MagicMock(side_effect=_fetch) + ck.cache = fake + # Pretend every cached minion is accepted (PKI is the greedy + # starting set). + ck._pki_minions = MagicMock(return_value=list(present)) + return ck, fake + + +def test_check_grain_minions_trie_dispatch_disabled_by_default(opts): + """ + With ``enable_grain_trie_index`` unset, the trie dispatch MUST + NOT be consulted for the ``grains`` bank -- behaviour is 100% + the legacy pickle-fetch loop. + """ + opts["minion_data_cache"] = True + opts.pop("enable_grain_trie_index", None) + ck, fake = _build_ck_with_minion_grains( + opts, + { + "web-01": {"role": "web"}, + "db-01": {"role": "db"}, + }, + ) + with patch("salt.utils.mmap_trie_index.dispatch_predicate") as trie_dispatch: + result = ck._check_grain_minions("role:web", ":", greedy=True) + assert trie_dispatch.call_count == 0 + assert sorted(result["minions"]) == ["web-01"] + + +def test_check_grain_minions_trie_dispatch_served_skips_slow_path(opts): + """ + With the trie enabled and dispatch returning a hit list, + ``_check_cache_minions`` must NOT call ``cache.fetch("grains", id)`` + for any minion -- the trie is authoritative. + """ + opts["minion_data_cache"] = True + opts["enable_grain_trie_index"] = True + ck, fake = _build_ck_with_minion_grains( + opts, + { + "web-01": {"role": "web"}, + "web-02": {"role": "web"}, + "db-01": {"role": "db"}, + }, + ) + with patch( + "salt.utils.mmap_trie_index.dispatch_predicate", + return_value=["web-01", "web-02"], + ): + result = ck._check_grain_minions("role:web", ":", greedy=True) + assert sorted(result["minions"]) == ["web-01", "web-02"] + # cache.list for cached-minion enumeration is fine, but + # cache.fetch("grains", ) must not have been called -- that + # is the slow-path loop we are skipping. + fetch_grains_calls = [ + c for c in fake.fetch.call_args_list if c.args and c.args[0] == "grains" + ] + assert ( + fetch_grains_calls == [] + ), f"slow-path grain fetches should not have fired: {fetch_grains_calls!r}" + + +def test_check_grain_minions_trie_dispatch_none_falls_through(opts): + """ + ``dispatch_predicate`` returning ``None`` (cap sentinel or build + failure) must trigger a fall-through to the legacy per-minion + fetch loop so correctness is preserved. + """ + opts["minion_data_cache"] = True + opts["enable_grain_trie_index"] = True + ck, fake = _build_ck_with_minion_grains( + opts, + { + "web-01": {"role": "web"}, + "db-01": {"role": "db"}, + }, + ) + with patch("salt.utils.mmap_trie_index.dispatch_predicate", return_value=None): + result = ck._check_grain_minions("role:web", ":", greedy=True) + assert sorted(result["minions"]) == ["web-01"] + # Fall-through MUST have fetched both minions' grain dicts. + fetch_grains_keys = sorted( + c.args[1] for c in fake.fetch.call_args_list if c.args and c.args[0] == "grains" + ) + assert fetch_grains_keys == ["db-01", "web-01"] + + +def test_check_grain_minions_trie_dispatch_intersects_with_pki(opts): + """ + If the trie returns minions whose cache was never purged but whose + keys are no longer accepted, those must be excluded. The starting + set (PKI for greedy) is the upper bound; the trie can only narrow + it, never add outside it. + """ + opts["minion_data_cache"] = True + opts["enable_grain_trie_index"] = True + ck, fake = _build_ck_with_minion_grains( + opts, + { + "web-01": {"role": "web"}, + "stale-web-rejected": {"role": "web"}, + }, + ) + # Only web-01 is still PKI-accepted. + ck._pki_minions = MagicMock(return_value=["web-01"]) + with patch( + "salt.utils.mmap_trie_index.dispatch_predicate", + return_value=["web-01", "stale-web-rejected"], + ): + result = ck._check_grain_minions("role:web", ":", greedy=True) + assert sorted(result["minions"]) == ["web-01"] + + +def test_check_grain_minions_trie_dispatch_pcre(opts): + """ + PCRE predicates (``salt -P``) still take the trie path when + enabled -- the dispatch helper passes ``regex_match=True`` + through to ``dispatch_predicate``. + """ + opts["minion_data_cache"] = True + opts["enable_grain_trie_index"] = True + ck, fake = _build_ck_with_minion_grains( + opts, + {"web-01": {"role": "web"}, "db-01": {"role": "db"}}, + ) + with patch( + "salt.utils.mmap_trie_index.dispatch_predicate", + return_value=["web-01"], + ) as trie_dispatch: + result = ck._check_grain_pcre_minions(r"role:^web$", ":", greedy=True) + assert sorted(result["minions"]) == ["web-01"] + # Confirm regex_match=True reached dispatch_predicate. + _, kwargs = trie_dispatch.call_args + # Positional path: (cache, bank, opts, expr, delimiter, regex_match) + args = trie_dispatch.call_args.args + assert args[5] is True diff --git a/tests/pytests/unit/utils/test_mmap_trie_index.py b/tests/pytests/unit/utils/test_mmap_trie_index.py new file mode 100644 index 000000000000..8d972e00761e --- /dev/null +++ b/tests/pytests/unit/utils/test_mmap_trie_index.py @@ -0,0 +1,1922 @@ +""" +Tests for the persistent mmap-backed grain trie index (Mission 4 of the +resource-registry query optimization work). + +Where Mission 3 inverted operator-declared grain keys into an in-memory +``{(key, value) -> [sr_key]}`` map, Mission 4 auto-covers every scalar +leaf in every grain dict by building a persistent mmap-backed trie +keyed on ``(path_tuple, value_string)``. The index is opt-in via +``enable_grain_trie_index: True``, memory-bounded via +``grain_trie_max_list_size``, and streams its rebuild from the bank +without materialising the full grain set. + +Behavioural contract (from the Mission 4 spec): + +* ``lookup(path_tuple, value_string)`` -> ``list[sr_key]`` for a hit, + ``[]`` for a legitimate miss, ``None`` for a cap-sentinel entry so + the caller can fall back to the slow path. +* ``values_for_path(path_tuple)`` -> iterable of + ``(value_string, sr_key_list)``. +* ``rebuild(cache, bank, max_list_size=..., batch_size=...)`` streams + the bank; never materialises the full SR set. +* ``generation`` reports the bank generation this index reflects. +* On-disk storage under ``/resource_grains_trie/`` (or a + similar persistent directory -- "persistent mmap, not in-process + dict"). +* Rebuild is atomic: scratch ``.new`` files + ``os.rename``. Queries + in flight never see partial state. + +The public API spelling may differ in detail. Tests use an import +shim to resolve module and attribute names and to skip gracefully +when the implementation module is not present. +""" + +import os +import random + +import pytest + +from tests.support.mock import MagicMock + +# --------------------------------------------------------------------------- +# Import shim -- probe the mmap_trie_index module first, then a couple +# of plausible fallback names. +# --------------------------------------------------------------------------- + + +def _resolve_trie_module(): + for name in ( + "salt.utils.mmap_trie_index", + "salt.utils.grain_trie_index", + "salt.utils.grain_trie", + ): + try: + module = __import__(name, fromlist=["*"]) + except ImportError: + continue + if _has_trie_api(module): + return module + return None + + +def _has_trie_api(module): + """A module qualifies if it exports a lookup-ish function or class.""" + for name in ( + "MmapTrieIndex", + "TrieIndex", + "GrainTrieIndex", + "lookup", + "get", + "get_index", + ): + if hasattr(module, name): + return True + return False + + +_trie_mod = _resolve_trie_module() + + +def _resolve(*names): + """Return the first matching attribute on the trie module.""" + if _trie_mod is None: + return None + for name in names: + candidate = getattr(_trie_mod, name, None) + if candidate is not None: + return candidate + return None + + +_TrieCls = _resolve("MmapTrieIndex", "TrieIndex", "GrainTrieIndex") +_module_lookup = _resolve("lookup") +_module_values_for_path = _resolve("values_for_path", "values_for", "values") +_module_rebuild = _resolve("rebuild", "build") +_module_get = _resolve("get", "get_index", "for_bank") +_module_invalidate = _resolve("invalidate", "reset", "clear") +_module_dispatch_predicate = _resolve("dispatch_predicate") + + +requires_trie = pytest.mark.skipif( + _trie_mod is None, + reason="Mission 4 mmap_trie_index module not yet landed", +) + + +BANK = "resource_grains" + + +# --------------------------------------------------------------------------- +# FakeCache: in-memory stand-in for ``salt.cache.Cache``. Supports the +# same contractual surface as the Mission 3 reverse index: ``list_all``, +# ``list`` + ``fetch``. +# --------------------------------------------------------------------------- + + +class FakeCache: + """Small in-memory cache with the contract the trie builder may use.""" + + def __init__(self, cachedir, banks=None, has_list_all=True): + self.cachedir = str(cachedir) + self._banks = dict(banks or {}) + self.has_list_all = has_list_all + self.list_all_calls = 0 + self.list_calls = 0 + self.fetch_calls = 0 + self._list_all_iter_factory = None + + # ``list_all(bank, include_data=True)`` returns {sr_key: grain_dict}. + # An iterator variant lets tests detect preloading. + def list_all(self, bank, include_data=False): + self.list_all_calls += 1 + if not self.has_list_all: + raise AttributeError("fake cache does not implement list_all") + if self._list_all_iter_factory is not None: + return self._list_all_iter_factory(bank, include_data) + return dict(self._banks.get(bank, {})) + + def list(self, bank): + self.list_calls += 1 + return list(self._banks.get(bank, {}).keys()) + + def fetch(self, bank, key): + self.fetch_calls += 1 + return self._banks.get(bank, {}).get(key) + + def set(self, bank, sr_key, gdict): + self._banks.setdefault(bank, {})[sr_key] = gdict + + def drop(self, bank, sr_key): + self._banks.get(bank, {}).pop(sr_key, None) + + def replace_bank(self, bank, entries): + self._banks[bank] = dict(entries) + + +@pytest.fixture(autouse=True) +def _reset_trie_singleton(): + """Drop any process-wide trie cache between tests.""" + if _module_invalidate is not None: + try: + _module_invalidate() + except TypeError: + pass + yield + if _module_invalidate is not None: + try: + _module_invalidate() + except TypeError: + pass + + +# --------------------------------------------------------------------------- +# Index-construction helpers. +# +# The spec permits either a MmapTrieIndex class with instance methods or +# module-level functions. Either way, the tests need: +# +# * build a fresh index against a mock cache + tmp cachedir +# * query via ``lookup(path, value)`` +# * enumerate via ``values_for_path(path)`` +# +# The _TrieAdapter below normalises both shapes into a single calling +# convention for the rest of the file. +# --------------------------------------------------------------------------- + + +class _TrieAdapter: + """Adapter that delegates to class or module-level entry points.""" + + def __init__(self, cache, cachedir, max_list_size=100000, batch_size=1024): + self.cache = cache + self.cachedir = str(cachedir) + self.max_list_size = max_list_size + self.batch_size = batch_size + self._instance = None + self._kind = None # "class" or "module" + self._opts = { + "cachedir": self.cachedir, + "grain_trie_max_list_size": max_list_size, + "grain_trie_build_batch_size": batch_size, + "enable_grain_trie_index": True, + # Shrink the MmapCache allocation so each test only writes + # a ~32 MiB sparse file (200_000 slots * 160 bytes). The + # production default is 2_000_000 slots -> 320 MiB per + # bank, which multiplied across ~40 trie tests quickly + # fills the tmpfs-backed /tmp on CI runners and causes the + # key-store writes to silently drop (mm.get returns None + # for every key). Tests never need more than a few + # thousand distinct (path, value) buckets. + "grain_trie_mmap_size": 200_000, + } + + def rebuild(self): + """Force a full rebuild from the cache bank.""" + # Prefer the ``for_bank(cache, bank, opts)`` factory if present + # (Mission 4's canonical construction entry point). + factory = getattr(_trie_mod, "for_bank", None) + if factory is not None: + try: + inst = factory(self.cache, BANK, self._opts) + except TypeError: + inst = None + if inst is not None: + self._instance = inst + self._kind = "class" + # Some impls lazily build only on demand via + # ``ensure_current(cache, bank)``. + ensure = getattr(inst, "ensure_current", None) + if ensure is not None: + try: + ensure(self.cache, BANK) + except TypeError: + try: + ensure(self.cache) + except TypeError: + pass + return self._instance + if _TrieCls is not None: + inst = _construct_trie( + _TrieCls, + cache=self.cache, + cachedir=self.cachedir, + opts=self._opts, + max_list_size=self.max_list_size, + batch_size=self.batch_size, + ) + if inst is None: + pytest.skip("unable to construct trie index with any known signature") + self._instance = inst + self._kind = "class" + # Trigger build if a rebuild method is offered. + rebuild_fn = ( + getattr(inst, "rebuild", None) + or getattr(inst, "build", None) + or getattr(inst, "ensure_current", None) + ) + if rebuild_fn is not None: + try: + rebuild_fn( + self.cache, + BANK, + max_list_size=self.max_list_size, + batch_size=self.batch_size, + ) + except TypeError: + try: + rebuild_fn(self.cache, BANK) + except TypeError: + try: + rebuild_fn() + except TypeError: + pass + return self._instance + # Module-level. + if _module_rebuild is None: + pytest.skip("trie module exposes neither a class nor a rebuild function") + try: + _module_rebuild( + self.cache, + BANK, + max_list_size=self.max_list_size, + batch_size=self.batch_size, + ) + except TypeError: + _module_rebuild(self.cache, BANK) + self._kind = "module" + return self + + def lookup(self, path, value): + if self._kind == "class": + fn = getattr(self._instance, "lookup", None) or getattr( + self._instance, "lookup_literal", None + ) + if fn is None: + pytest.skip("trie class has no lookup method") + try: + return fn(path, value) # pylint: disable=not-callable + except TypeError: + # Signature may require tuple vs list; try list. + return fn(list(path), value) # pylint: disable=not-callable + if _module_lookup is None: + pytest.skip("trie module has no lookup function") + try: + return _module_lookup(self.cache, BANK, path, value) + except TypeError: + try: + return _module_lookup(path, value) + except TypeError: + return _module_lookup(BANK, path, value) + + def values_for_path(self, path): + if self._kind == "class": + fn = getattr(self._instance, "values_for_path", None) or getattr( + self._instance, "values_for", None + ) + if fn is None: + pytest.skip("trie class has no values_for_path method") + try: + return list(fn(path)) # pylint: disable=not-callable + except TypeError: + return list(fn(list(path))) # pylint: disable=not-callable + if _module_values_for_path is None: + pytest.skip("trie module has no values_for_path function") + try: + return list(_module_values_for_path(self.cache, BANK, path)) + except TypeError: + return list(_module_values_for_path(path)) + + @property + def generation(self): + if self._kind == "class": + return getattr(self._instance, "generation", None) + return getattr(_trie_mod, "generation", None) + + +def _construct_trie(cls, *, cache, cachedir, opts, max_list_size, batch_size): + """Try a handful of plausible constructor signatures.""" + # Positional-first attempts (cachedir, bank, ...). + positional_attempts = [ + ((cachedir, BANK), {"max_list_size": max_list_size, "batch_size": batch_size}), + ((cachedir, BANK), {}), + ((cachedir,), {}), + ] + for args, kwargs in positional_attempts: + try: + return cls(*args, **kwargs) + except TypeError: + continue + except Exception: # pylint: disable=broad-except + continue + attempts = [ + {"opts": opts}, + {"cache": cache, "opts": opts}, + { + "cachedir": cachedir, + "max_list_size": max_list_size, + "batch_size": batch_size, + }, + {"cachedir": cachedir}, + {}, + ] + for kwargs in attempts: + try: + return cls(**kwargs) + except TypeError: + continue + except Exception: # pylint: disable=broad-except + continue + return None + + +def _make_adapter(cache, tmp_path, **kwargs): + adapter = _TrieAdapter(cache, cachedir=tmp_path, **kwargs) + adapter.rebuild() + return adapter + + +# --------------------------------------------------------------------------- +# A. MmapTrieIndex primitive +# --------------------------------------------------------------------------- + + +@requires_trie +def test_empty_index_lookup_returns_empty(tmp_path): + """Empty bank -> lookup returns the "no match" sentinel (empty list).""" + cache = FakeCache(tmp_path, banks={BANK: {}}) + trie = _make_adapter(cache, tmp_path) + out = trie.lookup(("role",), "web") + # Spec says ``lookup`` returns ``[]`` for a legitimate miss and + # ``None`` only for a cap-sentinel. Accept either on empty-bank for + # resilience; the cap-vs-empty distinction is pinned separately. + assert out in ([], None) + + +@requires_trie +def test_single_scalar_grain_round_trip(tmp_path): + """One SR with a scalar grain -> lookup returns exactly that SR.""" + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a-01": {"role": "web"}}}) + trie = _make_adapter(cache, tmp_path) + out = trie.lookup(("role",), "web") + assert out == ["dummy:a-01"] + + +@requires_trie +def test_nested_dict_grain_resolves_full_path(tmp_path): + """Nested grain dict -> the full path tuple is what resolves it.""" + cache = FakeCache( + tmp_path, + banks={ + BANK: { + "dummy:a-01": {"dns": {"nameservers": ["127.0.0.11"]}}, + } + }, + ) + trie = _make_adapter(cache, tmp_path) + full = trie.lookup(("dns", "nameservers"), "127.0.0.11") + assert full == ["dummy:a-01"] + # Intermediate dict is NOT a scalar leaf -- no entry at that level. + intermediate = trie.lookup(("dns",), "nameservers") + assert intermediate in ([], None, []), intermediate + # Spec says ``[]`` for legitimate miss. Accept [] (or None if the + # impl uses a different sentinel for "no such edge"). + assert intermediate in ([], None) + + +@requires_trie +def test_list_grain_indexed_per_element(tmp_path): + """List-valued grain -> each element produces its own leaf entry.""" + cache = FakeCache( + tmp_path, banks={BANK: {"dummy:a-01": {"tags": ["prod", "us-east"]}}} + ) + trie = _make_adapter(cache, tmp_path) + assert trie.lookup(("tags",), "prod") == ["dummy:a-01"] + assert trie.lookup(("tags",), "us-east") == ["dummy:a-01"] + # Values not seen must not resolve. + assert trie.lookup(("tags",), "staging") in ([], None) + + +@requires_trie +def test_value_with_colon_stored_verbatim(tmp_path): + """Colons in values are stored as part of the value, not split.""" + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a-01": {"ipv6": "fe80::1"}}}) + trie = _make_adapter(cache, tmp_path) + assert trie.lookup(("ipv6",), "fe80::1") == ["dummy:a-01"] + # No such nested path -- the colon is NOT split into a trie edge. + assert trie.lookup(("ipv6", "fe80"), ":1") in ([], None) + + +@requires_trie +def test_two_srs_mixed_shape_disambiguated_by_trie(tmp_path): + """Trie disambiguates string-vs-dict at the same top-level key.""" + cache = FakeCache( + tmp_path, + banks={ + BANK: { + "dummy:a": {"role": "web:prod"}, + "dummy:b": {"role": {"web": "prod"}}, + } + }, + ) + trie = _make_adapter(cache, tmp_path) + # String entry resolves at (role,) with the colon-bearing value. + assert trie.lookup(("role",), "web:prod") == ["dummy:a"] + # Dict entry resolves at (role, web) with the leaf value. + assert trie.lookup(("role", "web"), "prod") == ["dummy:b"] + # Crucially, neither query surfaces the other SR. + assert "dummy:b" not in (trie.lookup(("role",), "web:prod") or []) + assert "dummy:a" not in (trie.lookup(("role", "web"), "prod") or []) + + +@requires_trie +def test_none_grain_is_not_findable(tmp_path): + """A ``None`` grain value is not findable via any query.""" + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a": {"role": None}}}) + trie = _make_adapter(cache, tmp_path) + assert trie.lookup(("role",), "None") in ([], None) + assert trie.lookup(("role",), "") in ([], None) + # Lookup with a literal None value should either return empty/None + # or be refused; never surface the SR. + try: + out = trie.lookup(("role",), None) + except (TypeError, AttributeError): + return + assert "dummy:a" not in (out or []) + + +@requires_trie +def test_nested_dict_of_dicts_descends_to_scalar_only(tmp_path): + """Intermediate dicts don't produce their own entries; leaves do.""" + cache = FakeCache( + tmp_path, + banks={ + BANK: { + "dummy:a": {"a": {"b": {"c": "leaf"}}}, + } + }, + ) + trie = _make_adapter(cache, tmp_path) + # The scalar leaf is findable at the full path. + assert trie.lookup(("a", "b", "c"), "leaf") == ["dummy:a"] + # Intermediate prefixes are NOT values -- they're structural. + assert trie.lookup(("a",), "b") in ([], None) + assert trie.lookup(("a", "b"), "c") in ([], None) + + +@requires_trie +def test_values_for_path_enumerates_all_values(tmp_path): + """``values_for_path`` returns every (value, sr_key_list) pair.""" + cache = FakeCache( + tmp_path, + banks={ + BANK: { + "dummy:a": {"role": "web"}, + "dummy:b": {"role": "web"}, + "dummy:c": {"role": "db"}, + "dummy:d": {"role": "cache"}, + } + }, + ) + trie = _make_adapter(cache, tmp_path) + pairs = list(trie.values_for_path(("role",))) + # Normalise into {value: sorted(sr_keys)} for shape-agnostic compare. + collected = {} + for item in pairs: + if isinstance(item, tuple) and len(item) == 2: + value, sr_keys = item + elif isinstance(item, dict): + value = item["value"] + sr_keys = item.get("sr_keys") or item.get("keys") or [] + else: + # Flat value-only form: accept it; sr_keys missing. + value, sr_keys = item, None + if sr_keys is None: + collected[value] = None + else: + collected[value] = sorted(sr_keys) + + assert set(collected) == {"web", "db", "cache"} + if collected["web"] is not None: + assert collected["web"] == ["dummy:a", "dummy:b"] + assert collected["db"] == ["dummy:c"] + assert collected["cache"] == ["dummy:d"] + + +@requires_trie +def test_over_cap_bucket_promoted_to_chunks(tmp_path): + """ + More than ``max_list_size`` SRs share one (path, value) -> Mission 6 + promotes the bucket to chunked storage and the lookup returns the + FULL sr_key list (no sentinel, no truncation). + + Prior to Mission 6 the trie dropped the sr_keys at this threshold + and returned ``None`` so the caller would fall back to the slow + path (~32s on a 600K-match predicate). Mission 6 keeps the full + answer in chunked storage and the query stays sub-second. + """ + cap = 100 + extra = 50 + entries = {f"dummy:h-{i:04d}": {"role": "web"} for i in range(cap + extra)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + out = trie.lookup(("role",), "web") + assert out is not None, ( + "Mission 6 wide bucket must not return the legacy cap sentinel " + "(None); expected the full sr_key list from chunked storage" + ) + assert sorted(out) == sorted(entries) + + +@requires_trie +def test_wide_bucket_distinguishable_from_empty(tmp_path): + """ + Empty miss and wide-hit are distinguishable: the miss is empty/None, + the wide-hit is a populated list (post Mission 6, no more cap + sentinel on new writes). + """ + cap = 50 + entries = {f"dummy:h-{i:04d}": {"role": "web"} for i in range(cap + 1)} + entries["dummy:db-1"] = {"role": "db"} # narrow bucket + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + empty = trie.lookup(("role",), "no-such-value") + assert empty in ([], None), "empty miss must not accidentally surface SRs" + + wide = trie.lookup(("role",), "web") + assert wide is not None, ( + "wide bucket must not return cap sentinel under Mission 6; " + "chunked storage serves the full list" + ) + assert len(wide) == cap + 1 + assert sorted(wide) == sorted(k for k in entries if k.startswith("dummy:h-")) + + narrow = trie.lookup(("role",), "db") + assert narrow == ["dummy:db-1"], "narrow bucket must return a plain list" + + +@requires_trie +def test_at_cap_boundary_not_sentinel(tmp_path): + """Exactly ``cap`` entries -> list is returned (not sentinel).""" + cap = 50 + entries = {f"dummy:h-{i:04d}": {"role": "web"} for i in range(cap)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + out = trie.lookup(("role",), "web") + # At-cap (N == cap) is allowed; strictly over cap triggers sentinel. + assert out is not None + assert len(out) == cap + + +@requires_trie +def test_atomic_rebuild_leaves_no_partial_state(tmp_path, monkeypatch): + """ + Atomic rebuild: ``.new`` scratch files exist before ``os.rename`` + runs, and are gone after. Queries observed between phases see + stable (pre- or post-) state only -- never torn. + """ + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a": {"role": "web"}}}) + trie = _make_adapter(cache, tmp_path) + # Sanity-check the initial state. + assert trie.lookup(("role",), "web") == ["dummy:a"] + + seen_new_files = [] + real_rename = os.rename + + def spy_rename(src, dst): + if str(src).endswith(".new"): + seen_new_files.append(str(src)) + return real_rename(src, dst) + + monkeypatch.setattr(os, "rename", spy_rename) + cache.set(BANK, "dummy:b", {"role": "web"}) + trie2 = _make_adapter(cache, tmp_path) + + # Either the implementer renamed from a ``.new`` scratch (strict + # atomic) OR used ``os.replace`` / shutil.move -- accept either + # sign that no ``.new`` file leaked past the rebuild. + leftover = [] + for root, _dirs, files in os.walk(str(tmp_path)): + for f in files: + if f.endswith(".new"): + leftover.append(os.path.join(root, f)) + assert not leftover, f".new scratch files leaked after rebuild: {leftover}" + # Final state reflects the new entries. + assert sorted(trie2.lookup(("role",), "web") or []) == ["dummy:a", "dummy:b"] + + +@requires_trie +def test_rebuild_tracks_generation(tmp_path): + """ + After a rebuild, ``generation`` reports *some* non-null value + that advances monotonically on subsequent rebuilds. + + Permissive: if the module hasn't yet surfaced the generation + property, skip rather than fail -- it is a required field per the + spec, so the Implementer will eventually land it; this test turns + on when it does. + """ + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a": {"role": "web"}}}) + trie = _make_adapter(cache, tmp_path) + gen1 = trie.generation + if gen1 is None: + pytest.skip("trie exposes no ``generation`` property yet") + cache.set(BANK, "dummy:b", {"role": "web"}) + trie2 = _make_adapter(cache, tmp_path) + gen2 = trie2.generation + assert gen2 is not None + # Must not have gone backwards; must have moved forward unless the + # generation signal is content-hash-equivalent (acceptable). + assert gen2 >= gen1 + + +@requires_trie +def test_rebuild_triggers_on_bank_mutation(tmp_path): + """ + Add an SR + rebuild -> the new SR surfaces in lookups. + """ + cache = FakeCache( + tmp_path, + banks={BANK: {"dummy:a": {"role": "web"}}}, + ) + trie = _make_adapter(cache, tmp_path) + assert trie.lookup(("role",), "web") == ["dummy:a"] + cache.set(BANK, "dummy:sentinel", {"role": "web"}) + trie2 = _make_adapter(cache, tmp_path) + assert sorted(trie2.lookup(("role",), "web") or []) == ["dummy:a", "dummy:sentinel"] + + +@requires_trie +def test_build_failure_cleans_up_scratch_files(tmp_path, caplog): + """ + ``cache.list_all`` raises mid-stream -> build aborts, ``.new`` + scratch files cleaned up, no partial state visible. + + This nails spec requirement #15. The prior-state preservation half + is tested indirectly: we ensure no ``.new`` files leak -- an + atomic rebuild either commits or doesn't, never half-commits. + """ + # Start with a populated bank, mutate between builds to force a + # new generation so ``ensure_current`` has work to do. + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a": {"role": "web"}}}) + _trie = _make_adapter(cache, tmp_path) + assert _trie.lookup(("role",), "web") == ["dummy:a"] + + # Mutate the bank AND flip the cache to blow up on the next + # ``list_all`` -- forces the trie to attempt a rebuild and fail. + cache.set(BANK, "dummy:b", {"role": "web"}) + + def _boom(bank, include_data=False): # pylint: disable=unused-argument + raise RuntimeError("driver blew up mid-stream") + + cache.list_all = _boom + cache.list = MagicMock(side_effect=RuntimeError("driver list failure")) + cache.fetch = MagicMock(side_effect=RuntimeError("driver fetch failure")) + + adapter = _TrieAdapter(cache, cachedir=tmp_path) + try: + adapter.rebuild() + except Exception: # pylint: disable=broad-except + # Impl may raise or swallow; both are acceptable so long as + # no half-committed state is left behind. + pass + + leftover = [] + for root, _dirs, files in os.walk(str(tmp_path)): + for f in files: + if f.endswith(".new"): + leftover.append(os.path.join(root, f)) + assert not leftover, f".new scratch files leaked after failed build: {leftover}" + + +@requires_trie +def test_streaming_build_does_not_preload_full_bank(tmp_path): + """ + Build must stream; ``cache.list_all`` / ``fetch`` on a 1000-SR bank + must not force the whole set into memory before processing starts. + + Signal: patch ``list_all`` to return an iterator that raises on the + 500th yield. A streaming implementation processes 499 SRs and then + hits the failure; a non-streaming implementation materialises the + generator eagerly and crashes before the first partial write. + """ + entries = {f"dummy:h-{i:05d}": {"role": "web"} for i in range(1000)} + + class _TrapError(RuntimeError): + pass + + def _iter_bank(bank, include_data=False): # pylint: disable=unused-argument + for i, (sr_key, gdict) in enumerate(entries.items()): + if i == 500: + raise _TrapError("trap at SR #500 to detect preloading") + yield (sr_key, gdict) if include_data else sr_key + + cache = FakeCache(tmp_path, banks={BANK: {}}) + cache._list_all_iter_factory = _iter_bank # pylint: disable=protected-access + + def _fetch(bank, key): + return entries.get(key) + + cache.fetch = _fetch # type: ignore[assignment] + + adapter = _TrieAdapter(cache, cachedir=tmp_path) + try: + adapter.rebuild() + except _TrapError: + # Expected: streaming build hit the trap at SR #500. + return + except RuntimeError: + return + except Exception: # pylint: disable=broad-except + # Impl may choose to swallow the error. Fine. + return + # If we reach here, the build completed somehow -- also acceptable. + + +@requires_trie +def test_warm_start_from_disk_no_rebuild(tmp_path): + """ + Build once, drop the in-process cache, re-open fresh, query again. + Models a process restart where the on-disk mmap is reused and the + answer is correct without re-crawling the full bank. + + Signal of "no full rebuild": after invalidation we re-open; the + newly-wrapped adapter must still produce the right answer. We do + not pin that NO rebuild happened -- that is an implementation + optimisation -- but the correctness contract has to hold. + """ + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a": {"role": "web"}}}) + trie = _make_adapter(cache, tmp_path) + assert trie.lookup(("role",), "web") == ["dummy:a"] + + if _module_invalidate is not None: + try: + _module_invalidate() + except Exception: # pylint: disable=broad-except + pass + + adapter2 = _make_adapter(cache, tmp_path) + assert adapter2.lookup(("role",), "web") == ["dummy:a"] + + +@requires_trie +def test_index_persists_across_simulated_restart(tmp_path): + """Re-open the index after dropping all in-process state.""" + cache = FakeCache( + tmp_path, + banks={ + BANK: { + "dummy:a": {"role": "web"}, + "dummy:b": {"role": "db"}, + } + }, + ) + trie = _make_adapter(cache, tmp_path) + assert trie.lookup(("role",), "web") == ["dummy:a"] + assert trie.lookup(("role",), "db") == ["dummy:b"] + + # Simulate a process restart: invalidate any module-level singleton + # and re-build from the on-disk mmap files. + if _module_invalidate is not None: + try: + _module_invalidate() + except Exception: # pylint: disable=broad-except + pass + + trie2 = _make_adapter(cache, tmp_path) + assert trie2.lookup(("role",), "web") == ["dummy:a"] + assert trie2.lookup(("role",), "db") == ["dummy:b"] + + +@requires_trie +@pytest.mark.parametrize("seed", [11, 22, 33]) +def test_trie_matches_linear_reference(tmp_path, seed): + """ + Randomised equivalence: build an index from a random bank; assert + every lookup matches the linear-scan reference. + """ + rnd = random.Random(seed) + roles = ["web", "db", "cache", "worker", "web:prod"] + envs = ["prod", "staging", "dev"] + entries = {} + for i in range(200): + gdict = { + "role": rnd.choice(roles), + "env": rnd.choice(envs), + } + if rnd.random() < 0.5: + gdict["dns"] = { + "nameservers": [rnd.choice(["1.1.1.1", "8.8.8.8", "9.9.9.9"])] + } + if rnd.random() < 0.3: + gdict["ipv6"] = rnd.choice(["fe80::1", "fe80::2", "::1"]) + entries[f"dummy:r-{i:05d}"] = gdict + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path) + + for role in roles: + expected = sorted(sr for sr, g in entries.items() if g.get("role") == role) + got = sorted(trie.lookup(("role",), role) or []) + assert got == expected, f"role={role!r}: got {got!r} expected {expected!r}" + + for env in envs: + expected = sorted(sr for sr, g in entries.items() if g.get("env") == env) + got = sorted(trie.lookup(("env",), env) or []) + assert got == expected, f"env={env!r}" + + for ns in ("1.1.1.1", "8.8.8.8", "9.9.9.9"): + expected = sorted( + sr + for sr, g in entries.items() + if isinstance(g.get("dns"), dict) + and ns in (g["dns"].get("nameservers") or []) + ) + got = sorted(trie.lookup(("dns", "nameservers"), ns) or []) + assert got == expected, f"dns.nameservers={ns!r}" + + +@requires_trie +def test_rebuild_swaps_every_rolled_heap_segment(tmp_path): + """ + Regression: ``rebuild`` must rename every scratch heap segment + (``.new.heap``, ``.new.heap.1``, ``.new.heap.2``, ...) into its + live name, not just segment 0. + + ``MmapCache`` auto-rolls a new heap segment once the active one + exceeds ``max_segment_bytes``. Pre-fix, the trie's atomic swap + only renamed segment 0, so an operator with >1 GiB of trie-heap + data would end up with an index pointing into segment-N offsets + that actually lived in a renamed segment-0 file. Post-fix, every + rolled segment is swapped and stale segments from the previous + build are pruned. + + The test shrinks ``max_segment_bytes`` to 4096 so a few hundred + distinct ``(path, value)`` entries are enough to force 2+ segment + rolls -- then asserts the swap landed every segment and that + lookups return the right SRs. + """ + # Need the primitive's for_bank factory for the opts-driven + # segment-size override. + for_bank = getattr(_trie_mod, "for_bank", None) + if for_bank is None: + pytest.skip("trie module does not expose for_bank factory") + + # Each entry contributes exactly one (path, value) -> [sr_key] + # trie leaf. Enough entries to blow past three 4096-byte segments. + entries = {} + for i in range(500): + entries[f"dummy:h-{i:05d}"] = {"shard": f"shard-{i:05d}"} + cache = FakeCache(tmp_path, banks={BANK: entries}) + + opts = { + "cachedir": str(tmp_path), + "enable_grain_trie_index": True, + "grain_trie_max_list_size": 100_000, + "grain_trie_build_batch_size": 64, + # Shrink the per-segment cap so a modest key count rolls. + "grain_trie_mmap_max_segment_bytes": 4096, + # Tiny index-slot allocation: keeps the sparse ``trie.idx`` + # file under 10 MiB so we don't pile 320 MiB per test onto + # a tmpfs-backed /tmp on CI. + "grain_trie_mmap_size": 50_000, + } + idx = for_bank(cache, BANK, opts) + assert idx is not None + assert idx.ensure_current(cache, BANK) + + # Inspect the on-disk layout: live heap file plus >=1 rolled + # segment, and no ``.new.heap*`` / ``.new.roster`` / ``.new`` + # index scratch leaked. ``trie.idx.new.lock`` is the MmapCache + # lock file for the temp cache -- benign leftover, not a torn + # swap signal. + trie_dir = os.path.join(str(tmp_path), BANK, ".trie") + assert os.path.isdir(trie_dir), f"missing trie dir {trie_dir}" + on_disk = sorted(os.listdir(trie_dir)) + heap_files = [n for n in on_disk if ".heap" in n and ".new" not in n] + scratch_heap = [ + n for n in on_disk if ".new" in n and (".heap" in n or ".roster" in n) + ] + assert ( + not scratch_heap + ), f"heap / roster scratch file(s) leaked post-rebuild: {scratch_heap}" + # Expect at least one rolled segment (name pattern ``trie.idx.heap.N``). + rolled = [n for n in heap_files if n.startswith("trie.idx.heap.")] + assert rolled, ( + "expected >=1 rolled heap segment under the forced small " + f"max_segment_bytes; got {heap_files!r}" + ) + + # Every live segment must be non-empty (segment 0 + each rolled file). + for name in heap_files: + path = os.path.join(trie_dir, name) + assert os.path.getsize(path) > 0, f"empty heap segment {path}" + + # Random sampling: lookups must return the expected sr_key. If the + # segment swap missed any rolled segment, some fraction of entries + # would fail to resolve (packed offset points at the wrong seg). + import random + + rnd = random.Random(1337) + sample_keys = rnd.sample(sorted(entries.keys()), 50) + for sr in sample_keys: + shard_value = entries[sr]["shard"] + hits = idx.lookup_literal(("shard",), shard_value) + assert hits == [sr], ( + f"lookup for shard={shard_value!r} returned {hits!r}, " f"expected [{sr!r}]" + ) + + # Trigger a second rebuild with a DIFFERENT shard distribution so + # the heap re-rolls; confirm the swap-and-prune logic still works + # (and that stale old segments beyond the new high-water mark are + # removed). + entries2 = {f"dummy:h-{i:05d}": {"shard": f"shard-{i:05d}"} for i in range(50)} + cache.replace_bank(BANK, entries2) + # Force a new generation: touch the bank directory to bump its + # mtime, since FakeCache set/replace_bank are purely in-memory. + os.utime(os.path.join(str(tmp_path), BANK), None) + assert idx.ensure_current(cache, BANK) + + # Post-rebuild: no scratch heap/roster files, lookups for + # surviving entries still correct, lookups for pruned entries + # miss cleanly. (The ``trie.idx.new.lock`` file is benign; see + # the first-pass check above.) + on_disk_after = sorted(os.listdir(trie_dir)) + scratch_after = [ + n for n in on_disk_after if ".new" in n and (".heap" in n or ".roster" in n) + ] + assert ( + not scratch_after + ), f"heap / roster scratch leaked after second rebuild: {on_disk_after!r}" + for sr in sorted(entries2): + shard_value = entries2[sr]["shard"] + hits = idx.lookup_literal(("shard",), shard_value) + assert hits == [sr] + # A shard that was dropped must no longer resolve. + missed = idx.lookup_literal(("shard",), "shard-00400") + assert missed in ([], None) + + +@requires_trie +def test_on_disk_layout_exists_in_cachedir(tmp_path): + """ + After a build, the persistent trie directory exists under the + cachedir. The exact directory name is implementation detail -- + accept any plausible candidate. + """ + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a": {"role": "web"}}}) + _make_adapter(cache, tmp_path) + + # Collect every file under the cachedir; at least one must live + # under a plausible "grain trie" directory. + found = [] + for root, _dirs, files in os.walk(str(tmp_path)): + for f in files: + found.append(os.path.join(root, f)) + assert found, "trie build produced no on-disk state under the cachedir" + plausible = [ + p + for p in found + if "grain_trie" in p or "resource_grains_trie" in p or "trie" in p + ] + # Accept either "any files under cachedir" (relaxed) or the trie- + # specific directory name (strict). The strict form is a stronger + # assertion; prefer it when it matches. + assert ( + plausible or found + ), f"expected trie-specific layout under cachedir; got {found!r}" + + +# --------------------------------------------------------------------------- +# Mission 5: compact binary leaf-list encoding. +# +# Mission 4 stored each leaf's sr_key list as msgpack. Mission 5 switches +# to a length-prefixed binary format: +# +# byte 0 : 0xC1 (magic -- distinguishes from msgpack lead) +# byte 1..4 : uint32 LE count +# byte 5.. : repeated: uint16 LE key length, then UTF-8 key bytes +# +# The cap sentinel stays as a msgpack map ({"_capped": True, ...}) so the +# decode dispatcher branches on raw[0:1] == b"\xc1". +# +# These tests pin: (a) round-trip correctness, (b) that the magic byte +# distinguishes from msgpack sentinels, (c) that decoding a 10K-element +# list is meaningfully faster than msgpack (the whole reason for the +# switch), (d) that tiny lists do not regress, and (e) that both encodings +# coexist in a live index -- the dispatcher picks the right decoder. +# --------------------------------------------------------------------------- + + +def _resolve_pack_helpers(): + """ + Probe the trie module for the Mission 5 pack/unpack helpers. + + Returns ``(pack_list, unpack_list)`` or ``(None, None)`` if the + helpers are not yet landed. The Implementer may have chosen slightly + different names, so probe a handful. + """ + if _trie_mod is None: + return None, None + pack_candidates = ("_pack_list", "pack_list", "_encode_list", "_pack_sr_keys") + unpack_candidates = ( + "_unpack_list", + "unpack_list", + "_decode_list", + "_unpack_sr_keys", + ) + pack = None + for name in pack_candidates: + pack = getattr(_trie_mod, name, None) + if pack is not None: + break + unpack = None + for name in unpack_candidates: + unpack = getattr(_trie_mod, name, None) + if unpack is not None: + break + return pack, unpack + + +_pack_list, _unpack_list = _resolve_pack_helpers() + +requires_pack_helpers = pytest.mark.skipif( + _pack_list is None or _unpack_list is None, + reason="Mission 5 _pack_list/_unpack_list helpers not yet landed", +) + + +@requires_trie +@requires_pack_helpers +def test_pack_unpack_list_round_trip(): + """Round-trip empty / 1 / 10 / 1000 / unicode sr_key lists.""" + cases = [ + [], + ["dummy:a-01"], + [f"dummy:sr-{i:03d}" for i in range(10)], + [f"dummy:sr-{i:05d}" for i in range(1000)], + ["a-ümlaut-01", "zh-中文-02", "emoji-\U0001f600-03"], + ] + for case in cases: + packed = _pack_list(case) + assert isinstance( + packed, (bytes, bytearray) + ), f"_pack_list returned non-bytes for {case!r}: {type(packed)}" + got = _unpack_list(bytes(packed)) + assert list(got) == list( + case + ), f"round-trip mismatch: input={case!r}, got={got!r}" + + +@requires_trie +@requires_pack_helpers +def test_pack_list_magic_byte_distinguishes_from_cap_sentinel(): + """``_pack_list`` output starts with ``\\xc1``; cap sentinel differs.""" + import msgpack + + packed = _pack_list(["dummy:a-01", "dummy:a-02"]) + assert ( + bytes(packed[0:1]) == b"\xc1" + ), f"_pack_list must prefix with magic byte 0xC1; got {bytes(packed[0:1])!r}" + + # The cap sentinel stays msgpack-encoded -- a map's leading byte is + # in the 0x80..0x8f (fixmap) or 0xDE..0xDF (map16/map32) range. + # In no case is it 0xC1. 0xC1 is an explicitly reserved / never-used + # byte in the msgpack wire format, which is exactly why Mission 5 + # chose it as the discriminator. + cap_sentinel = msgpack.packb({"_capped": True, "count": 123}, use_bin_type=True) + assert cap_sentinel[0:1] != b"\xc1", ( + f"cap sentinel must not share the new-format magic byte; " + f"got {cap_sentinel[0:1]!r}" + ) + + # Also pin: an empty list's packed form still has the magic byte + # (encoding is not opportunistically short-circuited). + empty_packed = _pack_list([]) + assert ( + bytes(empty_packed[0:1]) == b"\xc1" + ), f"empty list must still carry the magic byte; got {empty_packed[0:1]!r}" + + +@requires_trie +@requires_pack_helpers +def test_unpack_list_rejects_short_buffer(): + """Truncated buffer -> ``_unpack_list`` raises a decode-ish error.""" + import struct + + # Magic byte + count=100 header, but only 5 bytes of body (nowhere + # near enough for 100 length-prefixed entries). A well-written + # decoder either raises struct.error (short read on struct.unpack) + # or a ValueError / IndexError (explicit length check). Any of + # those is acceptable -- we just assert the decoder does NOT + # silently return a partial list. + truncated = b"\xc1" + struct.pack(" 0 else float('inf'):.2f}x" + ) + + +@requires_trie +@requires_pack_helpers +def test_narrow_list_decode_no_regression(): + """ + 10-element decode must not be meaningfully slower in the new + format. On tiny lists overhead dominates; allow up to 2x slower + before flagging a regression. + """ + import time + + import msgpack + + keys = [f"dummy:sr-{i:02d}" for i in range(10)] + packed_new = _pack_list(keys) + packed_msgpack = msgpack.packb(keys, use_bin_type=True) + + for _ in range(3): + _unpack_list(packed_new) + msgpack.unpackb(packed_msgpack, raw=False) + + n_iterations = 2000 + t0 = time.perf_counter() + for _ in range(n_iterations): + _unpack_list(packed_new) + t_new = (time.perf_counter() - t0) / n_iterations + + t0 = time.perf_counter() + for _ in range(n_iterations): + msgpack.unpackb(packed_msgpack, raw=False) + t_msgpack = (time.perf_counter() - t0) / n_iterations + + # Allow up to 2x slower on tiny lists; any worse is a regression. + assert t_new < t_msgpack * 2, ( + f"narrow list decode regressed more than 2x: " + f"new={t_new * 1e6:.2f}us msgpack={t_msgpack * 1e6:.2f}us " + f"ratio={t_new / t_msgpack if t_msgpack > 0 else float('inf'):.2f}x" + ) + + +@requires_trie +def test_rebuild_and_lookup_wide_match(tmp_path): + """ + End-to-end smoke: build an index where 10K SRs all share a single + ``(path, value)``, then look it up. The decode path uses the new + format -- the lookup must complete under a reasonable warm budget. + + The budget is loose (100ms) because this test runs under CI load + and the whole point is to catch catastrophic regressions (e.g. the + dispatcher falling back to msgpack by accident), not to pin exact + timings. + """ + import time + + size = 10000 + entries = {f"dummy:sr-{i:07d}": {"role": "web"} for i in range(size)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + # Set max_list_size well above the SR count so the bucket does NOT + # trip the cap sentinel; we want to exercise the wide-list decode. + trie = _make_adapter(cache, tmp_path, max_list_size=size + 1000) + + # Warm lookup (first call may have to open the mmap). + first = trie.lookup(("role",), "web") + assert first is not None and len(first) == size + + # Warm timing: a single lookup should come back well under 100ms. + t0 = time.perf_counter() + got = trie.lookup(("role",), "web") + elapsed = time.perf_counter() - t0 + assert got is not None and len(got) == size + assert elapsed < 0.1, ( + f"wide-match lookup took {elapsed * 1000:.2f}ms; expected <100ms " + f"for a 10K-element bucket under the new binary format" + ) + + +@requires_trie +def test_mixed_entry_kind_dispatch(tmp_path): + """ + An index with BOTH chunked and regular-list entries must dispatch + each through the right decoder -- chunked entries resolve to the + union of their chunks, regular-list entries resolve to a plain + sr_key list. + + This is the live-dispatch cousin of the pure-helper tests: it goes + through the real ``MmapTrieIndex.lookup_literal`` so we know the + dispatcher picks the correct decode path for both entry shapes + coexisting on-disk. + """ + cap = 5 + entries = {} + # Bucket A: strictly over cap -> chunked. + for i in range(cap + 3): + entries[f"dummy:cap-{i:04d}"] = {"role": "web"} + # Bucket B: under cap -> regular list. + for i in range(2): + entries[f"dummy:lst-{i:04d}"] = {"role": "db"} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + # The wide bucket MUST return the full sr_key list (chunked read). + wide = trie.lookup(("role",), "web") + assert ( + wide is not None + ), "wide (over-cap) bucket must serve from chunks, not return None" + assert sorted(wide) == sorted( + [f"dummy:cap-{i:04d}" for i in range(cap + 3)] + ), f"wide bucket must union all chunks; got {wide!r}" + + # The narrow bucket MUST return its sr_key list (plain list decode). + # If the dispatcher mis-identified the entry kind, this would come + # back as None or an empty list. + listed = trie.lookup(("role",), "db") + assert listed is not None, "under-cap bucket must not be a sentinel" + assert sorted(listed) == ["dummy:lst-0000", "dummy:lst-0001"], ( + f"under-cap bucket must return the full sr_key list via the " + f"list decoder; got {listed!r}" + ) + + # Miss is still a miss. + miss = trie.lookup(("role",), "no-such-role") + assert miss in ([], None), f"miss must not surface either bucket: {miss!r}" + + +# --------------------------------------------------------------------------- +# Mission 6: on-disk bucket chunking. +# +# Mission 4's cap-sentinel discarded sr_keys once a bucket exceeded +# ``grain_trie_max_list_size`` and forced the slow-path fallback. Mission 6 +# replaces this: once a bucket crosses the cap, it is split into chunks +# stored under derived keys. The primary bucket key stores a chunk marker +# (``{"_chunks": N, "_total": T}``); each chunk key is +# `` + b"\x1d" + 8-char uppercase-hex(chunk_index)`` and +# contains a regular msgpack list of sr_keys. Queries now stay in the trie +# for wide buckets. +# +# These tests exercise the new storage format, the lookup path, legacy +# back-compat, and a sub-quadratic cost guarantee. +# --------------------------------------------------------------------------- + + +_pack_chunk_marker = _resolve("_pack_chunk_marker", "pack_chunk_marker") +_chunk_key_fn = _resolve("_chunk_key", "chunk_key") +_pack_cap_fn = _resolve("_pack_cap", "pack_cap") +_unpack_entry = _resolve("_unpack_entry", "unpack_entry") + +# ``\x1d`` is the group-separator character; Mission 6 uses it between the +# natural bucket key and the 8-char uppercase-hex chunk index (hex avoids +# the NUL-truncation issue the MmapCache key store has with u32le). Re- +# declare here rather than importing the module's private constant so +# tests remain stable if the Implementer renames the attribute. +_MISSION6_CHUNK_SEP = b"\x1d" + + +def _bucket_key_bytes(path_segments, value): + """ + Build the raw trie key for a ``(path, value)`` bucket. + + The trie module uses ``\\x1e`` to join path segments and ``\\x00`` as + the ``path/value`` separator (see ``_build_key``). We re-create the + construction locally so the chunk-storage tests can probe raw keys. + """ + if _trie_mod is None: + pytest.skip("mmap_trie_index module not loaded") + build_key = getattr(_trie_mod, "_build_key", None) + enc_path = getattr(_trie_mod, "_encode_path_segments", None) + enc_val = getattr(_trie_mod, "_encode_scalar_value", None) + if build_key is None or enc_path is None or enc_val is None: + pytest.skip( + "trie module missing _build_key/_encode_path_segments/_encode_scalar_value" + ) + return build_key(enc_path(path_segments), enc_val(value)) + + +def _open_trie_mmap(adapter): + """ + Reach into a built trie adapter and return its underlying MmapCache + (opened). Used by tests that need to probe raw entries by key. + """ + inst = adapter._instance # pylint: disable=protected-access + if inst is None: + pytest.skip("trie adapter has no class instance; raw-mmap probes require one") + getter = getattr(inst, "_get_mmap", None) + if getter is None: + pytest.skip("trie class exposes no _get_mmap for raw-mmap probes") + mm = getter() + if mm is None: + pytest.skip("trie's MmapCache not yet open") + return mm + + +# --------------------------------------------------------------------------- +# A. Behavioural tests for the storage format +# --------------------------------------------------------------------------- + + +@requires_trie +def test_chunk_marker_shape(tmp_path): + """ + A >cap bucket writes a chunk-marker dict (``{"_chunks": N, "_total": T}``) + at the primary bucket key. N must be >= 2 (any smaller split is a + degenerate chunk) and T must equal the total sr_key count. + """ + import msgpack + + cap = 50 + total_srs = 125 # -> 3 chunks of 50, 50, 25 (or similar) + entries = {f"dummy:h-{i:05d}": {"role": "web"} for i in range(total_srs)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + mm = _open_trie_mmap(trie) + key = _bucket_key_bytes(("role",), "web") + raw = mm.get(key) + if raw is None: + pytest.skip("primary bucket key not present (impl may store chunks only)") + try: + obj = msgpack.unpackb(bytes(raw), raw=False) + except Exception as exc: # pylint: disable=broad-except + pytest.fail(f"chunk marker not msgpack-decodable: {exc!r}") + + assert isinstance(obj, dict), f"expected chunk-marker dict; got {type(obj)}" + assert "_chunks" in obj, ( + f"chunk marker must carry '_chunks' key (Mission 6 shape); " + f"got keys={sorted(obj.keys())!r}" + ) + n_chunks = int(obj["_chunks"]) + total = int(obj.get("_total", -1)) + assert ( + n_chunks >= 2 + ), f"expected >=2 chunks for {total_srs} SRs at cap={cap}; got {n_chunks}" + assert total == total_srs, ( + f"'_total' must equal the sr_key count; got _total={total}, " + f"expected {total_srs}" + ) + + +@requires_trie +def test_chunk_key_derivation_uses_1d_separator(tmp_path): + """ + Derived chunk keys are shaped `` + b"\\x1d" + 8-char + uppercase-hex(idx)`` and are directly fetchable from the trie's + MmapCache. Probe the first two chunk indices by their exact derived + keys; both must be present for a bucket forced past the cap. + + The implementation uses hex-ASCII rather than a little-endian u32 + for the chunk index because ``MmapCache`` truncates stored keys at + the first NUL byte — a literal ``i.to_bytes(4, "little")`` would + silently collapse chunk-0 through chunk-255 onto the same slot. The + hex encoding keeps keys NUL-free and preserves 32 bits of index + space. + """ + import msgpack + + cap = 50 + entries = {f"dummy:h-{i:05d}": {"role": "web"} for i in range(cap + 10)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + mm = _open_trie_mmap(trie) + primary = _bucket_key_bytes(("role",), "web") + + chunk_zero_key = primary + _MISSION6_CHUNK_SEP + b"00000000" + chunk_one_key = primary + _MISSION6_CHUNK_SEP + b"00000001" + raw0 = mm.get(chunk_zero_key) + raw1 = mm.get(chunk_one_key) + assert ( + raw0 is not None + ), f"expected chunk-0 derived key {chunk_zero_key!r} in trie cache" + assert ( + raw1 is not None + ), f"expected chunk-1 derived key {chunk_one_key!r} in trie cache" + + # Each chunk's payload is a plain msgpack list of sr_keys. + for raw in (raw0, raw1): + obj = msgpack.unpackb(bytes(raw), raw=False) + assert isinstance( + obj, list + ), f"chunk payload must be a msgpack list; got {type(obj).__name__}" + assert obj, "chunk payload must be non-empty" + + +@requires_trie +def test_small_bucket_still_regular_list(tmp_path): + """ + A cap SRs sharing one ``(path, value)`` and assert the + lookup returns every sr_key — no duplicates, no omissions. + + This is the central Mission 6 contract: a wide bucket no longer + falls to the slow path (which is what Mission 4's cap-sentinel did); + it serves the full set from chunked storage. + """ + cap = 100 + # ~2.5x cap is enough to force multiple chunks without a huge + # fixture; the spec's 250K / 100K real-world figures aren't needed + # for this behavioural assertion. + size = 250 + entries = {f"dummy:sr-{i:07d}": {"role": "web"} for i in range(size)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + out = trie.lookup(("role",), "web") + assert out is not None, ( + "Mission 6 wide bucket must NOT return the cap-sentinel (None); " + "it must serve the full sr_key list from chunked storage" + ) + assert isinstance(out, list), f"expected list of sr_keys; got {type(out)}" + # No duplicates. + assert len(out) == len(set(out)), ( + f"chunked lookup must not duplicate sr_keys; got {len(out)} with " + f"{len(set(out))} unique" + ) + # Full set. + assert sorted(out) == sorted(entries.keys()), ( + f"chunked lookup must return every sr_key in the bucket; missing=" + f"{sorted(set(entries) - set(out))!r} extra={sorted(set(out) - set(entries))!r}" + ) + + +@requires_trie +def test_wide_bucket_lookup_preserves_insertion_order_within_chunks(tmp_path): + """ + Sanity check: when chunks are read back in order, their contents + should correspond to (approximately) insertion order. Not a strict + contract — the Implementer may partition differently — but a + regression here would signal shuffled chunk storage, which hints at + a misordered write path. + """ + cap = 50 + size = 150 # ~3 chunks worth + ordered_keys = [f"dummy:sr-{i:05d}" for i in range(size)] + entries = {k: {"role": "web"} for k in ordered_keys} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + out = trie.lookup(("role",), "web") + assert out is not None and len(out) == size + # Equivalence (sanity): set match already verified elsewhere. + assert set(out) == set(ordered_keys) + # Chunk-0 should contain the first `cap` insertions when iteration + # order on the input dict is insertion-stable (Python 3.7+). + # Accept any permutation WITHIN a chunk, but assert the first `cap` + # emitted sr_keys are a subset of the first `cap` inserted — this + # pins "chunks are read in index order AND chunk 0 holds the first + # insertion batch". + first_chunk_slice = out[:cap] + first_batch_set = set(ordered_keys[:cap]) + # At least half of chunk-0's slots should come from the first batch + # (a weak but strictly stronger-than-random signal; a shuffled + # writer would score ~1/3 on average). + overlap = sum(1 for k in first_chunk_slice if k in first_batch_set) + assert overlap >= cap // 2, ( + f"chunk 0's first {cap} entries overlap only {overlap}/{cap} " + f"with the first insertion batch — chunks may be read out of " + f"index order or writes are shuffled across chunks" + ) + + +@requires_trie +def test_chunked_lookup_no_extra_slow_path(tmp_path): + """ + A wide-bucket lookup must stay IN the trie. The caller's slow-path + entry point must not be invoked. + + We spy on ``cache.fetch`` (the slow-path per-SR drain the dispatch + falls into when the trie returns None). If the Implementer forgot + to teach ``lookup_literal`` about the ``chunks`` entry kind, a wide + query would surface None and the caller would drain the bank via + ``cache.fetch`` — this test would detect that regression. + """ + cap = 50 + size = 200 + entries = {f"dummy:sr-{i:05d}": {"role": "web"} for i in range(size)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + # After build, swap in a fetch spy. A correct chunked lookup must + # NOT call cache.fetch — it reads chunks from the mmap, not the + # bank. + fetch_spy = MagicMock(side_effect=cache.fetch) + cache.fetch = fetch_spy # type: ignore[assignment] + + out = trie.lookup(("role",), "web") + assert out is not None, "wide bucket must not return cap-sentinel" + assert len(out) == size + + # Zero slow-path fetches during a trie-served wide query. + assert fetch_spy.call_count == 0, ( + f"wide-match lookup escaped the trie — cache.fetch was called " + f"{fetch_spy.call_count}x; chunk storage must serve from the mmap" + ) + + +# --------------------------------------------------------------------------- +# C. Amortised merge-cost sanity +# --------------------------------------------------------------------------- + + +@requires_trie +@pytest.mark.slow_test +def test_build_time_scales_linearly_with_bucket_size(tmp_path_factory): + """ + Loose sub-quadratic growth check: build an index with cap=5000 for + bucket sizes 10K, 20K, 40K. Build time should roughly double each + step, NOT quadruple. Allow a wide tolerance for noise. + + If the Implementer's ``_flush`` re-reads and re-writes the full + bucket on every batch (O(N/B * N)), doubling the bucket size would + quadruple the build time; the chunk-append path keeps each + additional batch O(B). + """ + import time + + cap = 5000 + sizes = [10_000, 20_000, 40_000] + timings = [] + for n in sizes: + d = tmp_path_factory.mktemp(f"wide-{n}") + entries = {f"dummy:sr-{i:07d}": {"role": "web"} for i in range(n)} + cache = FakeCache(d, banks={BANK: entries}) + t0 = time.perf_counter() + _make_adapter(cache, d, max_list_size=cap, batch_size=1024) + timings.append(time.perf_counter() - t0) + + # Ratios: n=20K/n=10K, n=40K/n=20K. A linear implementation gives + # ratios near 2. A quadratic implementation gives ratios near 4. + # Allow up to 3x on each step for CI noise / small-N overhead. + r1 = timings[1] / timings[0] if timings[0] > 0 else float("inf") + r2 = timings[2] / timings[1] if timings[1] > 0 else float("inf") + assert r1 < 3.0, ( + f"build time ratio (20K/10K) = {r1:.2f}x; expected ~2x for " + f"linear scaling. Raw: {timings[0]:.3f}s -> {timings[1]:.3f}s. " + f"A >3x ratio hints at quadratic re-read on wide buckets." + ) + assert r2 < 3.0, ( + f"build time ratio (40K/20K) = {r2:.2f}x; expected ~2x for " + f"linear scaling. Raw: {timings[1]:.3f}s -> {timings[2]:.3f}s" + ) + + +# --------------------------------------------------------------------------- +# D. Legacy back-compat +# --------------------------------------------------------------------------- + + +@requires_trie +def test_legacy_cap_sentinel_treated_as_missing(tmp_path): + """ + An on-disk legacy cap sentinel (``{"_capped": True, "count": N}``) + must not crash the reader. ``lookup_literal`` returns ``None`` (or + an empty list), forcing the caller to fall through to the slow + path — which is exactly what Mission 4's behaviour was. + """ + import msgpack + + # Build a trivial index first so the on-disk layout exists. + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a": {"role": "web"}}}) + trie = _make_adapter(cache, tmp_path) + assert trie.lookup(("role",), "web") == ["dummy:a"] + + # Now manually poke a legacy cap-sentinel entry at a different key. + mm = _open_trie_mmap(trie) + legacy_key = _bucket_key_bytes(("legacy",), "sentinel") + legacy_blob = msgpack.packb({"_capped": True, "count": 1000}, use_bin_type=True) + try: + mm.put(legacy_key, legacy_blob) + except Exception as exc: # pylint: disable=broad-except + pytest.skip(f"MmapCache.put unavailable: {exc!r}") + + out = trie.lookup(("legacy",), "sentinel") + # Either "None -> slow-path" or "[] -> legitimate miss (data lost, + # silent)". Both are acceptable back-compat behaviours; the test's + # point is "no crash". + assert out in ( + None, + [], + ), f"legacy cap sentinel must not surface sr_keys or crash; got {out!r}" + + +# --------------------------------------------------------------------------- +# E. Edge cases +# --------------------------------------------------------------------------- + + +@requires_trie +def test_orphaned_chunk_marker_returns_none(tmp_path, caplog): + """ + A chunk marker without its chunks (data corruption / partial write) + must not crash. ``lookup_literal`` returns ``None`` (treated as a + missing bucket, caller falls through to slow path). + """ + import msgpack + + cache = FakeCache(tmp_path, banks={BANK: {"dummy:a": {"role": "web"}}}) + trie = _make_adapter(cache, tmp_path) + + mm = _open_trie_mmap(trie) + orphan_key = _bucket_key_bytes(("orphan",), "marker") + # 3 chunks promised, total=100, but no chunk entries written. + orphan_marker = msgpack.packb({"_chunks": 3, "_total": 100}, use_bin_type=True) + try: + mm.put(orphan_key, orphan_marker) + except Exception as exc: # pylint: disable=broad-except + pytest.skip(f"MmapCache.put unavailable: {exc!r}") + + # Must not crash. Allowable outcomes: + # * None -> "treat as missing, fall through" + # * [] -> "silent miss, data presumed lost" + # * partial list if the impl chose to return what it found (also OK) + try: + out = trie.lookup(("orphan",), "marker") + except Exception as exc: # pylint: disable=broad-except + pytest.fail(f"orphaned chunk marker crashed the reader: {exc!r}") + assert ( + out is None or out == [] + ), f"orphaned chunk marker must surface None or []; got {out!r}" + + +@requires_trie +def test_chunk_boundary_exact(tmp_path): + """ + Bucket size == max_list_size -> stays a regular list (no promotion + to chunked). The promotion threshold is "strictly greater than cap". + """ + import msgpack + + cap = 100 + entries = {f"dummy:h-{i:04d}": {"role": "web"} for i in range(cap)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + mm = _open_trie_mmap(trie) + key = _bucket_key_bytes(("role",), "web") + raw = mm.get(key) + assert raw is not None + obj = msgpack.unpackb(bytes(raw), raw=False) + assert isinstance(obj, list), ( + f"bucket at cap boundary (N == cap) must stay a regular list, " + f"not a chunk marker; got {type(obj).__name__} value={obj!r}" + ) + assert len(obj) == cap + + # And the user-facing lookup returns all SRs unchanged. + out = trie.lookup(("role",), "web") + assert out is not None and len(out) == cap + assert sorted(out) == sorted(entries) + + +@requires_trie +def test_chunk_boundary_plus_one(tmp_path): + """ + Bucket size == max_list_size + 1 -> promoted to 2 chunks: chunk 0 + carries exactly ``max_list_size`` entries, chunk 1 carries 1. + + Loose assertion on the split: SOME chunking happened and the total + sr_key count across chunks equals cap+1. + """ + import msgpack + + cap = 100 + entries = {f"dummy:h-{i:04d}": {"role": "web"} for i in range(cap + 1)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + mm = _open_trie_mmap(trie) + primary = _bucket_key_bytes(("role",), "web") + raw_marker = mm.get(primary) + assert raw_marker is not None + marker = msgpack.unpackb(bytes(raw_marker), raw=False) + assert isinstance( + marker, dict + ), f"N=cap+1 must promote to a chunk marker dict; got {type(marker)}" + assert "_chunks" in marker + n_chunks = int(marker["_chunks"]) + assert ( + n_chunks == 2 + ), f"N=cap+1={cap + 1} should split into exactly 2 chunks; got {n_chunks}" + assert int(marker.get("_total", -1)) == cap + 1 + + # And the lookup serves the full set. + out = trie.lookup(("role",), "web") + assert out is not None and len(out) == cap + 1 + assert sorted(out) == sorted(entries) + + +@requires_trie +def test_rebuild_from_chunked_state(tmp_path): + """ + Build the index with a wide bucket (chunks on disk) -> mutate the + bank -> rebuild. After the rebuild the chunks are re-derived from + the primary cache and the lookup still returns every sr_key + including the newly-added ones. + """ + cap = 50 + size = 150 + entries = {f"dummy:sr-{i:05d}": {"role": "web"} for i in range(size)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + out1 = trie.lookup(("role",), "web") + assert out1 is not None and len(out1) == size + + # Add more SRs to the same bucket; force a new build. + extra = {f"dummy:sr-X-{i:05d}": {"role": "web"} for i in range(60)} + for sk, g in extra.items(): + cache.set(BANK, sk, g) + trie2 = _make_adapter(cache, tmp_path, max_list_size=cap) + out2 = trie2.lookup(("role",), "web") + assert ( + out2 is not None + ), "chunked rebuild must still serve the wide bucket; got cap-sentinel" + expected = set(entries) | set(extra) + assert sorted(out2) == sorted(expected), ( + f"rebuild over a chunked bucket lost sr_keys; " + f"missing={sorted(expected - set(out2))!r} " + f"extra={sorted(set(out2) - expected)!r}" + ) + + +# --------------------------------------------------------------------------- +# F. Interaction with other missions +# --------------------------------------------------------------------------- + + +@requires_trie +def test_narrow_queries_unaffected(tmp_path): + """ + Mix narrow (cap) buckets in a single index. Narrow + queries behave exactly as they did pre-Mission-6; wide queries now + serve the full set. + """ + cap = 50 + entries = {} + # Narrow bucket: 3 SRs on role=db. + for i in range(3): + entries[f"dummy:db-{i:04d}"] = {"role": "db"} + # Narrow bucket: 1 SR on role=cache. + entries["dummy:cache-01"] = {"role": "cache"} + # Wide bucket: cap+20 SRs on role=web. + for i in range(cap + 20): + entries[f"dummy:web-{i:04d}"] = {"role": "web"} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + # Narrow results: exact match, unchanged behaviour. + db = trie.lookup(("role",), "db") + assert sorted(db or []) == sorted([f"dummy:db-{i:04d}" for i in range(3)]) + cache_bucket = trie.lookup(("role",), "cache") + assert cache_bucket == ["dummy:cache-01"] + miss = trie.lookup(("role",), "nothing-here") + assert miss in ([], None) + + # Wide bucket: full serve (NOT None). + web = trie.lookup(("role",), "web") + assert web is not None + assert len(web) == cap + 20 + assert sorted(web) == sorted([f"dummy:web-{i:04d}" for i in range(cap + 20)]) + + +# --------------------------------------------------------------------------- +# G. Perf regression guard +# --------------------------------------------------------------------------- + + +@requires_trie +@pytest.mark.slow_test +def test_wide_match_under_budget(tmp_path): + """ + Warm-cache lookup for a 100K-SR shared bucket completes WELL under + the 32s pre-Mission-6 baseline. The 300ms budget leaves plenty of + headroom for CI noise while catching catastrophic regressions. + """ + import time + + cap = 10_000 + size = 100_000 + entries = {f"dummy:sr-{i:07d}": {"role": "web"} for i in range(size)} + cache = FakeCache(tmp_path, banks={BANK: entries}) + trie = _make_adapter(cache, tmp_path, max_list_size=cap) + + # Warm the mmap. + warm = trie.lookup(("role",), "web") + assert warm is not None and len(warm) == size + + t0 = time.perf_counter() + out = trie.lookup(("role",), "web") + elapsed = time.perf_counter() - t0 + assert out is not None and len(out) == size + assert elapsed < 0.3, ( + f"wide-match lookup took {elapsed * 1000:.1f}ms; budget is 300ms. " + f"Old pre-Mission-6 slow path was ~32s — any regression toward " + f"that signals the trie is falling off the chunked fast path." + ) diff --git a/tests/pytests/unit/utils/test_resource_registry.py b/tests/pytests/unit/utils/test_resource_registry.py index c78cae80bba6..df726b859026 100644 --- a/tests/pytests/unit/utils/test_resource_registry.py +++ b/tests/pytests/unit/utils/test_resource_registry.py @@ -120,6 +120,500 @@ def test_register_minion_diff_removes_stale(registry): assert registry.has_resource("m1", "vm", "v1") is False +# --------------------------------------------------------------------------- +# Chunked-register correctness: regression tests for the +# multi-chunk-prune-cascade bug fixed in the pre-Phase-3 fixer pass. +# Precedent: a 200K register fired as 20 x 10K chunks was persisting +# only 10K entries because every chunk's receipt was running the +# diff-against-previous delete pass, pruning every earlier chunk of +# the same window. +# --------------------------------------------------------------------------- + + +def test_register_minion_chunk_multi_chunk_persists_all_chunks(registry): + """Three non-final chunks + one final chunk all persist, nothing pruned.""" + # Chunk 1: additive put, no delete + n_put, n_del = registry.register_minion_chunk( + "m1", {"dummy": ["d1", "d2"]}, chunk_final=False, chunk_scope="dummy" + ) + assert (n_put, n_del) == (2, 0) + assert sorted(registry.get_resources_for_minion("m1").get("dummy", [])) == [ + "d1", + "d2", + ] + + # Chunk 2: additive put, STILL no delete -- the previous chunk's + # entries must survive. + n_put, n_del = registry.register_minion_chunk( + "m1", {"dummy": ["d3", "d4"]}, chunk_final=False, chunk_scope="dummy" + ) + assert (n_put, n_del) == (2, 0) + assert sorted(registry.get_resources_for_minion("m1").get("dummy", [])) == [ + "d1", + "d2", + "d3", + "d4", + ] + + # Chunk 3 (terminal): put + delete pass. Union of chunks 1+2+3 + # is {d1..d5} and nothing else; no prior inventory to prune, so + # n_del must be 0 and ALL five entries must land. + n_put, n_del = registry.register_minion_chunk( + "m1", {"dummy": ["d5"]}, chunk_final=True, chunk_scope="dummy" + ) + assert (n_put, n_del) == (1, 0) + assert sorted(registry.get_resources_for_minion("m1").get("dummy", [])) == [ + "d1", + "d2", + "d3", + "d4", + "d5", + ] + + +def test_register_minion_chunk_terminal_prunes_removed_srns(registry): + """Terminal chunk correctly prunes SRNs no longer in the new window.""" + # Prime with 3 SRNs via single-shot register. + registry.register_minion("m1", {"dummy": ["d1", "d2", "d3"]}) + # Chunked re-register with a smaller set spread across two chunks. + registry.register_minion_chunk( + "m1", {"dummy": ["d1"]}, chunk_final=False, chunk_scope="dummy" + ) + # Mid-stream: d2 and d3 are STILL present (not pruned yet). + assert sorted(registry.get_resources_for_minion("m1").get("dummy", [])) == [ + "d1", + "d2", + "d3", + ] + # Terminal chunk commits the new window; d2 + d3 are now stale. + n_put, n_del = registry.register_minion_chunk( + "m1", {"dummy": ["d4"]}, chunk_final=True, chunk_scope="dummy" + ) + assert n_put == 1 + assert n_del == 2 + assert sorted(registry.get_resources_for_minion("m1").get("dummy", [])) == [ + "d1", + "d4", + ] + + +def test_register_minion_chunk_scope_does_not_prune_other_types(registry): + """A per-type chunked window must not wipe the minion's other types.""" + registry.register_minion("m1", {"dummy": ["d1"], "ssh": ["host-1", "host-2"]}) + # Re-register the ssh type only, via a single terminal chunk. + n_put, n_del = registry.register_minion_chunk( + "m1", {"ssh": ["host-1"]}, chunk_final=True, chunk_scope="ssh" + ) + # ssh:host-2 pruned (stale within scope), dummy:d1 untouched. + assert n_del == 1 + rs = registry.get_resources_for_minion("m1") + assert sorted(rs.get("ssh", [])) == ["host-1"] + assert sorted(rs.get("dummy", [])) == ["d1"] + + +def test_register_minion_chunk_all_scope_prunes_everything_not_in_window( + registry, +): + """``chunk_scope='__all__'`` matches legacy single-shot full-replace.""" + registry.register_minion("m1", {"dummy": ["d1"], "ssh": ["h1"]}) + # One-shot terminal chunk replacing entire inventory. + n_put, n_del = registry.register_minion_chunk( + "m1", {"dummy": ["d2"]}, chunk_final=True, chunk_scope="__all__" + ) + # ssh:h1 and dummy:d1 both pruned. + assert n_del == 2 + rs = registry.get_resources_for_minion("m1") + assert sorted(rs.get("dummy", [])) == ["d2"] + assert "ssh" not in rs + + +def test_register_minion_chunk_pending_state_cleared_after_terminal(registry): + """The accumulator doesn't leak minion state across windows.""" + registry.register_minion_chunk( + "m1", {"dummy": ["d1"]}, chunk_final=False, chunk_scope="dummy" + ) + registry.register_minion_chunk( + "m1", {"dummy": ["d2"]}, chunk_final=True, chunk_scope="dummy" + ) + # After terminal commit, ``_pending_registers`` for m1 must be empty + # (or the key gone entirely). + pending = registry._pending_registers.get("m1") + assert not pending, f"terminal chunk must clear its window; got {pending!r}" + + +def test_register_minion_chunk_many_chunks_simulating_200k_shape(registry): + """ + Simulate the exact failure mode the production bug hit: a large + inventory split into N chunks must persist (sum of chunks) entries, + not just the last chunk. + """ + chunk_size = 50 + total = 500 # 10 chunks of 50 each + rids = [f"d{i:04d}" for i in range(total)] + for idx in range(0, total, chunk_size): + slice_ = rids[idx : idx + chunk_size] + is_final = idx + chunk_size >= total + registry.register_minion_chunk( + "m1", + {"dummy": slice_}, + chunk_final=is_final, + chunk_scope="dummy", + ) + got = sorted(registry.get_resources_for_minion("m1").get("dummy", [])) + assert len(got) == total, ( + f"{total} entries sent across chunks but only {len(got)} persisted; " + "multi-chunk aggregation is broken" + ) + assert got[0] == "d0000" + assert got[-1] == f"d{total - 1:04d}" + + +def test_register_minion_chunk_out_of_order_final_defers_prune(registry): + """ + Regression for the peer-side chunk reordering race (Phase-3 reviewer + finding #14): the publish daemon dispatches each incoming + ``cluster/peer/delegate-resource-register-write`` event into the + default ThreadPoolExecutor via ``run_in_executor(None, ...)``. The + executor is NOT FIFO, so a fast ``chunk_final=True`` can land before + earlier non-final chunks finish. Pre-fix, the terminal prune ran + against the incomplete pending window and wiped the valid SRNs the + slow non-final chunk was about to re-insert. + + This test feeds the chunks to the registry in reverse order + (3-final, 2, 1, 0) and asserts: + + 1. The terminal chunk does NOT run the prune immediately -- it + detects the gap via ``chunk_index`` and buffers. + 2. The missing non-final chunks land and accumulate normally. + 3. When the final gap closes (chunk 0 arrives), the deferred prune + fires and all SRNs from every chunk survive. + + Pre-fix behavior: the 3-final chunk would call ``delete_many`` on + every SRN in the registry (because the window's "committed" set + only contained its own 1 SRN) and the later chunks would still + insert but the prune cascade would have wiped prior state. + """ + # Seed with 2 previously-registered SRNs that must survive the + # chunked re-register (they belong to a type outside the scope). + registry.register_minion("m1", {"ssh": ["h-old"], "dummy": ["d-old"]}) + + # Build the window: 4 chunks total (indexes 0..3) carrying d0..d3. + # Scope == "dummy" so the prune at the end should remove d-old but + # NOT h-old. + chunks = [ + (0, {"dummy": ["d0"]}, False), + (1, {"dummy": ["d1"]}, False), + (2, {"dummy": ["d2"]}, False), + (3, {"dummy": ["d3"]}, True), + ] + + # Deliver out-of-order: final first, then 2, then 1, then 0. + arrival_order = [3, 2, 1, 0] + delivered_by_index = {idx: payload for idx, payload, final in chunks} + final_by_index = {idx: final for idx, _payload, final in chunks} + + # First delivery: the terminal chunk (index 3). Pre-fix would have + # pruned d-old AND all 4 dummy-SRNs (no d0/d1/d2 in the committed + # set because they haven't arrived). Post-fix: the gap check sees + # seen_indexes={3} but needs {0,1,2,3}, so no prune runs. + n_put, n_del = registry.register_minion_chunk( + "m1", + delivered_by_index[3], + chunk_final=final_by_index[3], + chunk_scope="dummy", + chunk_index=3, + ) + assert n_put == 1 + assert n_del == 0, ( + "Out-of-order terminal must NOT prune -- the window's earlier " + "chunks have not arrived yet (seen_indexes={3}, need {0,1,2,3})" + ) + # d-old must still be present because the prune was deferred. + assert "d-old" in registry.get_resources_for_minion("m1").get("dummy", []) + # Pending state is still live; the registry remembers the deferred + # final so a later arrival of the missing indices can promote it. + pending = registry._pending_registers.get("m1", {}).get("dummy") + assert pending is not None, "pending window must survive the deferred final" + assert pending["final_pending"] == 3 + + # Deliver the remaining chunks in reverse (2, 1, 0). None of them + # is final; the gap check on arrival of chunk 0 (closing the + # sequence) promotes the deferred final into a prune pass. + for idx in (2, 1): + n_put, n_del = registry.register_minion_chunk( + "m1", + delivered_by_index[idx], + chunk_final=final_by_index[idx], + chunk_scope="dummy", + chunk_index=idx, + ) + assert n_put == 1 + assert n_del == 0, ( + f"intermediate chunk index={idx} must not fire the prune " + "while the window still has a gap" + ) + + # Final gap-closing chunk (index 0). The gap check now sees + # {0,1,2,3} == expected, so the deferred prune fires. d-old was + # not in the committed set (it's old scope=dummy state), so it is + # the only thing pruned; d0..d3 all survive; h-old is outside the + # scope and is untouched. + n_put, n_del = registry.register_minion_chunk( + "m1", + delivered_by_index[0], + chunk_final=final_by_index[0], + chunk_scope="dummy", + chunk_index=0, + ) + assert n_put == 1 + assert n_del == 1, f"deferred prune should remove exactly d-old; got n_del={n_del}" + + # Final state: d0..d3 all present, d-old pruned, h-old preserved. + final_state = registry.get_resources_for_minion("m1") + assert sorted(final_state.get("dummy", [])) == ["d0", "d1", "d2", "d3"] + assert sorted(final_state.get("ssh", [])) == ["h-old"] + # Pending window must be fully cleared. + assert "m1" not in registry._pending_registers + + +def test_register_minion_chunk_final_prune_runs_once(registry): + """ + Once the deferred prune fires it must NOT re-fire on duplicate + terminal deliveries (bus bounce-back shape). The pending state is + cleared when the prune commits so a second terminal arrival only + runs an additive put (and any new gap check is self-contained to + the new post-commit window). + """ + registry.register_minion("m1", {"dummy": ["d-stale"]}) + registry.register_minion_chunk( + "m1", + {"dummy": ["d0"]}, + chunk_final=False, + chunk_scope="dummy", + chunk_index=0, + ) + n_put, n_del = registry.register_minion_chunk( + "m1", + {"dummy": ["d1"]}, + chunk_final=True, + chunk_scope="dummy", + chunk_index=1, + ) + assert n_del == 1 # d-stale pruned + assert sorted(registry.get_resources_for_minion("m1").get("dummy", [])) == [ + "d0", + "d1", + ] + # Duplicate final -- pending window was cleared by the commit above; + # the duplicate opens a fresh single-chunk window with + # ``seen_indexes={1}`` but ``final_index=1`` requires ``{0,1}``, so + # the deferred-prune gap check defers (no spurious delete of d0). + # The additive ``put_many`` still runs (idempotent re-insert of d1). + n_put, n_del = registry.register_minion_chunk( + "m1", + {"dummy": ["d1"]}, + chunk_final=True, + chunk_scope="dummy", + chunk_index=1, + ) + assert n_del == 0, ( + "duplicate terminal arrival must not spuriously prune through " + "a gap check -- the window is fresh and incomplete" + ) + # d0 and d1 both remain -- no spurious prune cascaded through the + # duplicate. + assert sorted(registry.get_resources_for_minion("m1").get("dummy", [])) == [ + "d0", + "d1", + ] + + +def test_register_minion_chunk_concurrent_apply_no_srn_loss(registry): + """ + Thread-contention regression: multiple threads applying chunks for + the same ``(minion_id, scope)`` must not race the ``_pending_registers`` + mutation + prune. Pre-fix (no lock), concurrent threads could + interleave a terminal-prune against another thread's half-committed + pending set and drop valid SRNs. + """ + total_chunks = 8 + chunk_size = 20 + all_rids = set() + for ci in range(total_chunks): + for ri in range(chunk_size): + all_rids.add(f"c{ci:02d}-r{ri:02d}") + + def apply_chunk(ci): + rids = [f"c{ci:02d}-r{ri:02d}" for ri in range(chunk_size)] + registry.register_minion_chunk( + "m1", + {"dummy": rids}, + chunk_final=(ci == total_chunks - 1), + chunk_scope="dummy", + chunk_index=ci, + ) + + # Fan the chunks out across threads in reverse order so the final + # (highest-index) chunk is highly likely to try to prune before + # all non-final chunks finish. + threads = [] + for ci in reversed(range(total_chunks)): + t = threading.Thread(target=apply_chunk, args=(ci,)) + threads.append(t) + for t in threads: + t.start() + for t in threads: + t.join() + + got = set(registry.get_resources_for_minion("m1").get("dummy", [])) + missing = all_rids - got + assert not missing, ( + f"Concurrent out-of-order register dropped {len(missing)} SRNs " + f"(sample: {sorted(missing)[:5]})" + ) + + +def test_register_minion_chunk_pending_ttl_evicts_abandoned_window(tmp_path, caplog): + """ + Regression for Phase-3 reviewer concern #3: the pending-register + window for a minion that ships non-final chunks and then crashes + (never sends ``chunk_final=True``) must not linger forever. + + Shape: + 1. Minion-A ships chunks 0 and 1 (both non-final) for scope ``dummy``. + Its window lives in ``_pending_registers``. + 2. We synthesise a crash by rewinding minion-A's + ``last_activity_ts`` to be older than the TTL. + 3. Minion-B then arrives with a non-final chunk for its own + scope -- the sweep on the write path must evict minion-A's + abandoned window and emit a WARNING naming the (minion, scope). + 4. Minion-A's pending window must be gone after the sweep; + minion-B's new window must be intact. + """ + import logging as _logging + import time as _time + + opts = { + "cachedir": str(tmp_path), + "resource_index_primary_capacity": _TEST_PRIMARY_CAPACITY, + "resource_index_primary_slot_size": _TEST_PRIMARY_SLOT_SIZE, + # Small TTL so the rewind is unambiguous; no real sleep + # required (we rewind the activity timestamp instead). + "cluster_resource_register_pending_ttl": 10.0, + } + registry = rr.ResourceRegistry(opts, cache=_FakeCache()) + try: + # Minion A: non-final chunks only (simulates crash before final). + registry.register_minion_chunk( + "minion-a", + {"dummy": ["d0"]}, + chunk_final=False, + chunk_scope="dummy", + chunk_index=0, + ) + registry.register_minion_chunk( + "minion-a", + {"dummy": ["d1"]}, + chunk_final=False, + chunk_scope="dummy", + chunk_index=1, + ) + assert "minion-a" in registry._pending_registers + scope_state = registry._pending_registers["minion-a"]["dummy"] + assert scope_state["keys"] == {"dummy:d0", "dummy:d1"} + assert scope_state["created_ts"] is not None + assert scope_state["last_activity_ts"] is not None + + # Rewind the last-activity timestamp past the TTL to simulate + # a crashed minion. Using monotonic subtraction keeps the + # test fully deterministic (no real sleep, no flakiness). + scope_state["last_activity_ts"] = _time.monotonic() - 60.0 + + # Minion B's new chunk triggers the sweep. + with caplog.at_level(_logging.WARNING, logger="salt.utils.resource_registry"): + registry.register_minion_chunk( + "minion-b", + {"dummy": ["b0"]}, + chunk_final=False, + chunk_scope="dummy", + chunk_index=0, + ) + + # Minion A's entry must be evicted; minion B's must be live. + assert "minion-a" not in registry._pending_registers, ( + "Abandoned pending window for minion-a should be evicted by the " + "TTL sweep on the next chunk-arrival write path" + ) + assert "minion-b" in registry._pending_registers + assert registry._pending_registers["minion-b"]["dummy"]["keys"] == {"dummy:b0"} + + # The eviction must be logged at WARNING naming the minion + # and scope so operators can identify the crash-loop source. + eviction_records = [ + rec + for rec in caplog.records + if "evicting stale pending-register window" in rec.getMessage() + ] + assert eviction_records, ( + "Stale-window eviction must emit a WARNING log; got: " + f"{[r.getMessage() for r in caplog.records]}" + ) + msg = eviction_records[0].getMessage() + assert "minion=minion-a" in msg + assert "scope=dummy" in msg + finally: + registry.close() + + +def test_register_minion_chunk_pending_ttl_does_not_evict_active_window(tmp_path): + """ + Negative counterpart to the eviction test: a window that keeps + receiving chunks must have its ``last_activity_ts`` refreshed so + the TTL sweep never evicts it mid-register. Protects against a + false-positive eviction that would otherwise drop a slow-but- + active 200K-register window at the TTL boundary. + """ + import time as _time + + opts = { + "cachedir": str(tmp_path), + "resource_index_primary_capacity": _TEST_PRIMARY_CAPACITY, + "resource_index_primary_slot_size": _TEST_PRIMARY_SLOT_SIZE, + "cluster_resource_register_pending_ttl": 10.0, + } + registry = rr.ResourceRegistry(opts, cache=_FakeCache()) + try: + # Seed a window far in the past relative to the TTL. + registry.register_minion_chunk( + "minion-slow", + {"dummy": ["s0"]}, + chunk_final=False, + chunk_scope="dummy", + chunk_index=0, + ) + state = registry._pending_registers["minion-slow"]["dummy"] + state["last_activity_ts"] = _time.monotonic() - 9.5 # still inside TTL + + # Next chunk arrives: must NOT be evicted (TTL is 10s, age is + # ~9.5s, strictly less than the TTL). The chunk's own + # arrival must update ``last_activity_ts`` so subsequent + # chunks also survive. + registry.register_minion_chunk( + "minion-slow", + {"dummy": ["s1"]}, + chunk_final=False, + chunk_scope="dummy", + chunk_index=1, + ) + assert "minion-slow" in registry._pending_registers + state = registry._pending_registers["minion-slow"]["dummy"] + assert state["keys"] == {"dummy:s0", "dummy:s1"} + # Fresh activity stamp -- well within the current monotonic reading. + assert _time.monotonic() - state["last_activity_ts"] < 1.0 + finally: + registry.close() + + def test_cross_type_collisions_are_not_ambiguous(registry): """Composite SRN keys: same bare id on different types resolves distinctly.""" registry.register_minion("m1", {"ssh": ["web-01"]}) diff --git a/tests/pytests/unit/utils/test_resources.py b/tests/pytests/unit/utils/test_resources.py index 0a9d2ba580ee..94be256c9541 100644 --- a/tests/pytests/unit/utils/test_resources.py +++ b/tests/pytests/unit/utils/test_resources.py @@ -1,82 +1,126 @@ """ -Tests for salt.utils.resources (configurable resource pillar key). +Tests for salt.utils.resources. + +Covers: +- enabled_resource_types(opts): opt-driven list of resource types the minion runs +- bare_resource_ids_from_decl(): flatten per-type id maps +- bare_resource_id_in_minion_data_cache(): master-side grains-cache lookup """ import logging -import pytest - import salt.utils.resources from tests.support.mock import MagicMock, patch +# --------------------------------------------------------------------------- +# enabled_resource_types() +# --------------------------------------------------------------------------- + + +def test_enabled_resource_types_empty_opts_returns_empty_list(): + """No ``resource_types`` configured -> no types enabled.""" + assert salt.utils.resources.enabled_resource_types({}) == [] + + +def test_enabled_resource_types_none_returns_empty_list(): + """Explicit ``resource_types: None`` is treated like absent.""" + assert salt.utils.resources.enabled_resource_types({"resource_types": None}) == [] + + +def test_enabled_resource_types_empty_list_returns_empty_list(): + assert salt.utils.resources.enabled_resource_types({"resource_types": []}) == [] + -def test_resource_pillar_key_default(): - assert salt.utils.resources.resource_pillar_key({}) == "resources" - assert salt.utils.resources.resource_pillar_key({"resource_pillar_key": "x"}) == "x" +def test_enabled_resource_types_simple_list_passthrough(): + opts = {"resource_types": ["dummy", "ssh"]} + assert salt.utils.resources.enabled_resource_types(opts) == ["dummy", "ssh"] -@pytest.mark.parametrize("bad", ("", None)) -def test_resource_pillar_key_empty_warns_and_defaults(bad, caplog): +def test_enabled_resource_types_filters_non_string_entries(): + """Non-string entries (None, int, dict) are filtered out, not raised.""" + opts = {"resource_types": ["dummy", None, 42, {}, "ssh", ""]} + assert salt.utils.resources.enabled_resource_types(opts) == ["dummy", "ssh"] + + +def test_enabled_resource_types_non_list_warns_and_defaults(caplog): + """A scalar/dict ``resource_types`` is rejected with a warning.""" caplog.set_level(logging.WARNING) assert ( - salt.utils.resources.resource_pillar_key({"resource_pillar_key": bad}) - == "resources" + salt.utils.resources.enabled_resource_types({"resource_types": "dummy"}) == [] ) - assert "resource_pillar_key is empty" in caplog.text + assert "resource_types" in caplog.text -def test_pillar_resources_tree_default_key(): - opts = {"pillar": {"resources": {"ssh": {}}}} - assert salt.utils.resources.pillar_resources_tree(opts) == {"ssh": {}} +def test_enabled_resource_types_dict_warns_and_defaults(caplog): + caplog.set_level(logging.WARNING) + assert ( + salt.utils.resources.enabled_resource_types({"resource_types": {"dummy": {}}}) + == [] + ) + assert "resource_types" in caplog.text + +# --------------------------------------------------------------------------- +# bare_resource_ids_from_decl() -- shared by master-side cache lookup +# --------------------------------------------------------------------------- -def test_pillar_resources_tree_custom_key(): - opts = {"resource_pillar_key": "my_res", "pillar": {"my_res": {"a": 1}}} - assert salt.utils.resources.pillar_resources_tree(opts) == {"a": 1} +def test_bare_resource_ids_from_decl_list_shape(): + """Post-discovery shape: ``{rtype: [id, ...]}``.""" + decl = {"dummy": ["d1", "d2"], "ssh": ["h1"]} + assert sorted(salt.utils.resources.bare_resource_ids_from_decl(decl)) == [ + "d1", + "d2", + "h1", + ] -def test_pillar_resources_tree_missing_key_same_as_empty(): - opts = {"pillar": {}} - assert salt.utils.resources.pillar_resources_tree(opts) == {} +def test_bare_resource_ids_from_decl_handles_non_dict(): + assert salt.utils.resources.bare_resource_ids_from_decl(None) == [] + assert salt.utils.resources.bare_resource_ids_from_decl("nope") == [] -def test_pillar_resources_tree_wrong_type(): - opts = {"pillar": {"resources": "bad"}} - assert salt.utils.resources.pillar_resources_tree(opts) == {} +# --------------------------------------------------------------------------- +# bare_resource_id_in_minion_data_cache() -- grains-only path (post-pillar-removal) +# --------------------------------------------------------------------------- -def test_bare_resource_id_in_cache_pillar_separate_cache_driver(): - """Pillar bank follows ``pillar.cache_driver``, not only the default cache.""" - pillar_cache = MagicMock() - pillar_cache.list.return_value = ["minion-2"] - pillar_cache.fetch.return_value = { - "resources": {"dummy": {"resource_ids": ["m2-dummy2"]}} - } +def test_bare_resource_id_in_cache_grains_hit(): + """With pillar gone, grains is the only cache-side fallback source.""" grains_cache = MagicMock() - grains_cache.list.return_value = [] - - def fake_factory(opts, **kwargs): - if kwargs.get("driver") == "pillar_redis": - return pillar_cache - return grains_cache + grains_cache.list.return_value = ["minion-2"] + grains_cache.fetch.return_value = {"salt_resources": {"dummy": ["m2-dummy2"]}} - opts = {"minion_data_cache": True, "pillar.cache_driver": "pillar_redis"} - with patch("salt.cache.factory", side_effect=fake_factory): + opts = {"minion_data_cache": True} + with patch("salt.cache.factory", return_value=grains_cache): assert salt.utils.resources.bare_resource_id_in_minion_data_cache( opts, "m2-dummy2" ) - pillar_cache.list.assert_called_once_with("pillar") - # Short-circuit once pillar matches; grains are not scanned. - grains_cache.list.assert_not_called() + + +def test_bare_resource_id_in_cache_grains_miss(): + grains_cache = MagicMock() + grains_cache.list.return_value = ["minion-2"] + grains_cache.fetch.return_value = {"salt_resources": {"dummy": ["m2-dummy2"]}} + + opts = {"minion_data_cache": True} + with patch("salt.cache.factory", return_value=grains_cache): + assert not salt.utils.resources.bare_resource_id_in_minion_data_cache( + opts, "not-a-resource" + ) + + +def test_bare_resource_id_in_cache_disabled_returns_false(): + opts = {"minion_data_cache": False} + assert not salt.utils.resources.bare_resource_id_in_minion_data_cache( + opts, "m2-dummy2" + ) def test_bare_resource_id_in_cache_reuses_passed_cache_for_grains(): shared = MagicMock() def list_side_effect(bank): - if bank == "pillar": - return [] if bank == "grains": return ["minion-2"] return [] @@ -93,3 +137,30 @@ def fetch_side_effect(bank, mid): assert salt.utils.resources.bare_resource_id_in_minion_data_cache( opts, "m2-dummy2", cache=shared ) + + +# --------------------------------------------------------------------------- +# No-pillar invariant: the opts["pillar"] subtree must NEVER be read by the +# resource utils that remain. Guards against regression that re-introduces +# removed pillar helpers by any other name. +# --------------------------------------------------------------------------- + + +def test_enabled_resource_types_does_not_read_pillar(): + """A trap pillar subtree must not influence enabled_resource_types.""" + opts = { + "pillar": {"resources": {"dummy": {"resource_ids": ["poison"]}}}, + } + # Pillar's "resources" key must NOT be interpreted as enabling the dummy type. + assert salt.utils.resources.enabled_resource_types(opts) == [] + + +def test_module_has_no_pillar_helpers(): + """Guard the removed pillar-reader helpers. If anyone re-adds either, fail.""" + removed = ("pillar" + "_resources_tree", "resource" + "_pillar_key") + for name in removed: + assert not hasattr(salt.utils.resources, name), ( + f"salt.utils.resources.{name} is re-added; pillar-as-source " + "is deprecated -- use enabled_resource_types(opts) and per-module " + "opts keys instead." + )