From 5db04cd8043797e474ecb699eddc86a21f04146c Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 5 Nov 2026 20:47:29 +0000 Subject: [PATCH] retry cgroup discovery after a failed `/proc` read Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- src/proclimits/_cgroup.py | 192 ++++++++++++++++++++---- src/proclimits/_sensor.py | 108 ++++++++++---- src/proclimits/_types.py | 42 +++++- src/proclimits/_windows.py | 17 ++- tests/unit/conftest.py | 18 ++- tests/unit/test_cgroup.py | 289 ++++++++++++++++++++++++++++++++++--- tests/unit/test_sensor.py | 250 ++++++++++++++++++++++++++++++-- tests/unit/test_windows.py | 31 +++- 9 files changed, 843 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index 51fa07b..fb1837e 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Inside a container the usual answers describe the machine, not the process. `psu Reading the cgroup files directly is not enough either. The limit that applies is rarely written where the process is, and a cgroup that restricts nothing does not say so by leaving its limit file empty. A consumer that reads one file and takes it at face value believes it has 8 EB of memory. -This package handles both, and reports only what actually restricts the process. `None` means nothing visible to the process restricts it - the machine is then the honest answer, and `get_machine_cpu_count()` and `get_machine_memory_bytes()` below give it. +This package handles both, and reports only what actually restricts the process. `None` means nothing visible to the process restricts it, or a limit could not be read or used. `describe().notices` says which. The machine is then what is left to size from, and `get_machine_cpu_count()` and `get_machine_memory_bytes()` below give it. Off Linux and Windows every limit reads as `None`. macOS still reports the memory and the CPU cores of the machine. On other platforms the machine facts can read as `None`. Some examples below pair this package with `psutil`, which it does not require - `psutil` answers what the machine is using, which is not a question about limits. @@ -193,6 +193,8 @@ A limit of `None` with no notice about it means nothing went wrong: the mechanis | `CPU_USAGE_SCOPE_MISMATCH` | the level the CPU limit applies to counts no CPU time, so no rate can be measured | cgroup v1 with `cpu` and `cpuacct` on separate mounts, where the level holding the quota has no counterpart under the accounting one | | `MEMORY_METRICS_UNAVAILABLE` | no mechanism here is present to carry a memory limit | a host with no cgroup filesystem mounted, or a platform this package has no backend for | | `CPU_METRICS_UNAVAILABLE` | no mechanism here is present to carry a CPU limit | the same | +| `MEMORY_METRICS_UNREADABLE` | where a memory limit would be read from cannot be located, because a `/proc` file cannot be read; the next reading tries again | a `/proc` file that could not be read while no discovery is kept: before the first one succeeded, or after `clear_cache()` or a fork | +| `CPU_METRICS_UNREADABLE` | the same for a CPU limit | the same | | `MEMORY_ANCESTORS_HIDDEN` | only part of the memory hierarchy is visible, so a limit above it is enforced but unreadable | a container given a subtree as its whole `/sys/fs/cgroup` | | `CPU_ANCESTORS_HIDDEN` | only part of a CPU hierarchy is visible | the same | | `MEMORY_LIMIT_UNREADABLE` | a level holds a memory limit that says nothing usable, so what it enforces is unknown | a hand-built or mocked cgroup tree; no kernel writes such a value | diff --git a/src/proclimits/_cgroup.py b/src/proclimits/_cgroup.py index 4610aef..1761177 100644 --- a/src/proclimits/_cgroup.py +++ b/src/proclimits/_cgroup.py @@ -1,7 +1,8 @@ from __future__ import annotations +import errno import os -from dataclasses import dataclass +from dataclasses import dataclass, replace from functools import lru_cache from pathlib import Path, PurePosixPath @@ -152,6 +153,24 @@ def __init__(self, directory: Path) -> None: """The level that could not be read.""" +class _UnreadableProcFileError(Exception): + """A `/proc` file that discovery reads failed for a reason other than its absence. + + The error the read raised may name no file. One raised after the file was opened does not, and neither does + a `ValueError`. So this error prints as the file and the reason. The reason is the errno name where there is + one, and the kind of error otherwise. + """ + + def __init__(self, path: Path, error: OSError | ValueError) -> None: + # The errno tells a process out of file descriptors, EMFILE, from a file nobody may read, EACCES. + reason = ( + errno.errorcode.get(error.errno, str(error.errno)) + if isinstance(error, OSError) and error.errno is not None + else type(error).__name__ + ) + super().__init__(f'{path} ({reason})') + + @dataclass(frozen=True) class _Mount: """One line of the mount table, as far as this module cares.""" @@ -226,7 +245,8 @@ class Controllers: """The controllers that carry resource metrics, as located for this process.""" interface: Interface | None - """The interface holding this process, whatever it carries. `None` where no hierarchy holds it at all. + """The interface holding this process, whatever it carries. `None` where no hierarchy holds it at all, or + where none can be located right now. Named once for the machine rather than per metric, so that a metric no hierarchy carries is still reported against the mechanism that would have carried it. A located controller names its own interface. @@ -244,6 +264,24 @@ class Controllers: cpu_set: Controller | None """Carries the set of cores the cgroup may run on.""" + unlocated_file: str | None + """The `/proc` file that could not be read, and why, as ` ()`. `None` where none failed. + + Set, nothing is located, and the next lookup tries again. + """ + + +_NO_CONTROLLERS = Controllers( + interface=None, + memory=None, + cpu_quota=None, + cpu_usage=None, + cpu_set=None, + unlocated_file=None, +) +"""Nothing located. Discovery answers this where `/proc` is not there, and a failed lookup adds the file it could +not read.""" + @dataclass(frozen=True) class _MemoryUsage: @@ -297,9 +335,13 @@ def available(self) -> int | None: limit_level=None, unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) """What is reported where no level holds a memory limit at all.""" +_NO_CPU = RawCpu(quota=None, cpu_set=None, unreadable_level=None, unconvertible_level=None, unlocated_file=None) +"""Nothing read. A reading that fails adds what it could not read.""" + def read_memory() -> RawMemory: """Read the tightest memory limit along the chain, with a usage that keeps the distance to it. @@ -307,8 +349,17 @@ def read_memory() -> RawMemory: An out-of-memory kill follows from how much memory can still be allocated, not from a ratio. The smallest distance to a limit anywhere along the chain is therefore the quantity carried over, expressed against the tightest limit so that a consumer sees one comparable pair. + + Where a `/proc` file keeps the controller from being located, nothing is read and `unlocated_file` names it. """ - controller = locate_controllers().memory + controllers = locate_controllers() + + # A failed lookup locates no controller, as a machine without one does. The reading names the file, so the + # two stay apart even where a later lookup succeeds. + if controllers.unlocated_file is not None: + return replace(_NO_MEMORY, unlocated_file=controllers.unlocated_file) + + controller = controllers.memory if controller is None: return _NO_MEMORY @@ -323,6 +374,7 @@ def read_memory() -> RawMemory: limit_level=None, unreadable_level=str(unreadable.directory), usage_unreadable_level=None, + unlocated_file=None, ) if not levels: @@ -344,6 +396,7 @@ def read_memory() -> RawMemory: limit_level=str(tightest.directory), unreadable_level=None, usage_unreadable_level=str(min_distance_level.directory), + unlocated_file=None, ) return RawMemory( @@ -354,6 +407,7 @@ def read_memory() -> RawMemory: limit_level=str(tightest.directory), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -385,21 +439,34 @@ def _read_memory_levels(controller: Controller) -> list[_MemoryLevel]: def read_cpu() -> RawCpu: - """Read both CPU restrictions, and the level that said nothing usable if there was one.""" + """Read both CPU restrictions, and the level that said nothing usable if there was one. + + Where a `/proc` file keeps the controllers from being located, nothing is read and `unlocated_file` names it. + """ + # One lookup for both readings, and neither looks up on its own. A quota whose lookup failed beside a set + # whose lookup recovered would report the set, which can be looser. + controllers = locate_controllers() + if controllers.unlocated_file is not None: + return replace(_NO_CPU, unlocated_file=controllers.unlocated_file) + try: return RawCpu( - quota=read_cpu_quota(), - cpu_set=read_cpu_set_size(), + quota=read_cpu_quota(controllers), + cpu_set=read_cpu_set_size(controllers), unreadable_level=None, unconvertible_level=None, + unlocated_file=None, ) except _UnreadableFileError as unreadable: - return RawCpu(quota=None, cpu_set=None, unreadable_level=str(unreadable.directory), unconvertible_level=None) + return replace(_NO_CPU, unreadable_level=str(unreadable.directory)) -def read_cpu_quota() -> RawCpuQuota | None: +def read_cpu_quota(controllers: Controllers) -> RawCpuQuota | None: """Read the tightest CPU bandwidth quota along the chain, and the level it binds at. + Args: + controllers: The lookup this reading uses. + Returns: The quota, or `None` when no level sets one. Taken as read: a quota can exceed the cores of the machine. @@ -408,7 +475,7 @@ def read_cpu_quota() -> RawCpuQuota | None: _UnreadableFileError: If a level holds a quota file that says nothing usable. `read_cpu` turns that into a reading of nothing. """ - controller = locate_controllers().cpu_quota + controller = controllers.cpu_quota if controller is None: return None @@ -424,13 +491,16 @@ def read_cpu_quota() -> RawCpuQuota | None: return RawCpuQuota( cores=cores, limit_level=str(directory), - usage_level=_cpu_usage_level(directory, controller), + usage_level=_cpu_usage_level(directory, controller, controllers.cpu_usage), ) -def read_cpu_set_size() -> RawCpuSet | None: +def read_cpu_set_size(controllers: Controllers) -> RawCpuSet | None: """Read the set of CPU cores the cgroup of this process may run on. + Args: + controllers: The lookup this reading uses. + Returns: The set, or `None` when no level sets one. Taken as read: a set can cover every core of the machine. @@ -438,7 +508,7 @@ def read_cpu_set_size() -> RawCpuSet | None: _UnreadableFileError: If the level holds a set that says nothing usable. `read_cpu` turns that into a reading of nothing. """ - controller = locate_controllers().cpu_set + controller = controllers.cpu_set if controller is None: return None @@ -457,7 +527,7 @@ def read_cpu_set_size() -> RawCpuSet | None: return RawCpuSet( cores=cores, limit_level=str(directory), - usage_level=_cpu_usage_level(directory, controller), + usage_level=_cpu_usage_level(directory, controller, controllers.cpu_usage), ) @@ -524,19 +594,34 @@ def read_cpu_usage(level: str | None = None) -> float | None: return nanoseconds / _NANOSECONDS_PER_SECOND if nanoseconds is not None else None -@lru_cache(maxsize=1) def locate_controllers() -> Controllers: """Locate the control files carrying the resource metrics of this process. - Discovery walks `/proc` and is cached until `clear_cache()` forgets it. The control files themselves are - read again on every sample. + Discovery walks `/proc`, and what it finds is kept until `clear_cache()` forgets it. A `/proc` file that + cannot be read is not kept: nothing is located, the file is named, and the next lookup tries again. The + control files themselves are read again on every sample. + """ + try: + return _discover_controllers() + except _UnreadableProcFileError as unreadable: + return replace(_NO_CONTROLLERS, unlocated_file=str(unreadable)) + + +@lru_cache(maxsize=1) +def _discover_controllers() -> Controllers: + """Locate the controllers by walking `/proc`, and keep the answer. + + `lru_cache` keeps no exception, so a failure raised from here is read again on the next lookup. + + Raises: + _UnreadableProcFileError: If a `/proc` file that discovery reads fails for a reason other than its absence. """ # Each metric is located on its own, because a machine can serve them through different interfaces. try: hierarchies = _read_hierarchies() - except (OSError, ValueError): - # Not Linux, or `/proc` is not mounted. Nothing to read either way. - return Controllers(interface=None, memory=None, cpu_quota=None, cpu_usage=None, cpu_set=None) + except (FileNotFoundError, NotADirectoryError): + # Not Linux, or `/proc` is not mounted. That lasts as long as the process, so it is kept. + return _NO_CONTROLLERS return Controllers( interface=_interface(hierarchies), @@ -544,6 +629,7 @@ def locate_controllers() -> Controllers: cpu_quota=_locate_controller(hierarchies, _CPU_QUOTA), cpu_usage=_locate_cpu_usage(hierarchies), cpu_set=_locate_controller(hierarchies, _CPU_SET), + unlocated_file=None, ) @@ -577,6 +663,7 @@ def sources() -> Sources: cpu_quota=_source(controllers.cpu_quota, controllers.interface), cpu_set=_source(controllers.cpu_set, controllers.interface), cpu_usage=_source(controllers.cpu_usage, controllers.interface), + unlocated_file=controllers.unlocated_file, ) @@ -584,7 +671,8 @@ def _source(controller: Controller | None, interface: Interface | None) -> Sourc """Spell one metric as a source.""" if controller is None: # A hierarchy that carries no such controller searched no levels, and `None` is kept for the machine - # that has no hierarchy at all - the sensor layer reads that as "nothing here can say". + # where no hierarchy is located at all - the sensor layer reads that as "nothing here can say", or as + # "cannot be located right now" where `unlocated_file` is set. return Source(interface=interface, levels=()) if interface is not None else None return Source( @@ -685,7 +773,7 @@ def _machine_cpu_count_fallback() -> int | None: def clear_cache() -> None: """Forget the located controllers. The next reading discovers them again.""" - locate_controllers.cache_clear() + _discover_controllers.cache_clear() # A child of `fork` inherits what its parent discovered. A pre-fork server whose supervisor puts each worker @@ -730,14 +818,13 @@ def _bound_controllers(hierarchy: _Hierarchy) -> list[str]: return [] -def _cpu_usage_level(directory: Path, source: Controller) -> str | None: +def _cpu_usage_level(directory: Path, source: Controller, usage: Controller | None) -> str | None: """Find where the CPU time of one level is counted, given that level in another hierarchy. Under cgroup v1 the quota and the accounting can be two hierarchies with two mount points, so the level is translated by its path below the mount point. Every chain ends at its own mount point, which is what - makes that path comparable. + makes that path comparable. `usage` is the counter located by the same lookup as `source`. """ - usage = locate_controllers().cpu_usage if usage is None: return None @@ -879,7 +966,9 @@ def _read_hierarchies() -> _Hierarchies: """Locate the mounted cgroup hierarchies and the cgroup this process belongs to in each. Raises: - OSError: If `/proc/self/mountinfo` or `/proc/self/cgroup` cannot be read. + FileNotFoundError: If `/proc/self/mountinfo` or `/proc/self/cgroup` is not there. + NotADirectoryError: If a directory on the path to either is not one. + _UnreadableProcFileError: If either cannot be read for any other reason. """ unified_path, controller_paths = _read_own_paths() mounts = _read_mounts() @@ -947,11 +1036,11 @@ def _read_mounts() -> list[_Mount]: """Read the mount table, skipping the lines that are not mount entries. Raises: - OSError: If `/proc/self/mountinfo` cannot be read. + FileNotFoundError: If `/proc/self/mountinfo` is not there. + NotADirectoryError: If a directory on its path is not one. + _UnreadableProcFileError: If it cannot be read for any other reason. """ - # `errors='replace'` because a foreign mount point with non-UTF-8 bytes must not hide our own cgroup. - # Such a path decodes to something that matches no file, so only that mount drops out. - lines = _PROC_SELF_MOUNTINFO.read_text(errors='replace').splitlines() + lines = _read_proc_file(_PROC_SELF_MOUNTINFO).splitlines() return [mount for line in lines if (mount := _parse_mount(line)) is not None] @@ -978,12 +1067,14 @@ def _read_own_paths() -> tuple[str | None, dict[str, str]]: """Read the cgroup this process belongs to in the unified hierarchy and in each cgroup v1 one. Raises: - OSError: If `/proc/self/cgroup` cannot be read. + FileNotFoundError: If `/proc/self/cgroup` is not there. + NotADirectoryError: If a directory on its path is not one. + _UnreadableProcFileError: If it cannot be read for any other reason. """ unified: str | None = None controllers: dict[str, str] = {} - for line in _PROC_SELF_CGROUP.read_text(errors='replace').splitlines(): + for line in _read_proc_file(_PROC_SELF_CGROUP).splitlines(): try: _hierarchy_id, controller_list, cgroup_path = line.split(':', 2) except ValueError: @@ -1001,6 +1092,45 @@ def _read_own_paths() -> tuple[str | None, dict[str, str]]: return unified, controllers +def _read_proc_file(path: Path) -> str: + """Read a `/proc` file that discovery depends on, telling an absent one from one that cannot be read. + + Raises: + FileNotFoundError: If the file is not there. + NotADirectoryError: If a directory on its path is not one. + _UnreadableProcFileError: If it cannot be read for any other reason. It names the file. + """ + try: + # `errors='replace'` because a foreign mount point with non-UTF-8 bytes must not hide our own cgroup. + # Such a path decodes to something that matches no file, so only that mount drops out. + return path.read_text(errors='replace') + except (FileNotFoundError, NotADirectoryError): + raise + except (OSError, ValueError) as error: + # Out of file descriptors, opening an absent file fails with EMFILE rather than ENOENT. + if _is_absent(path): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), str(path)) from error + + raise _UnreadableProcFileError(path, error) from error + + +def _is_absent(path: Path) -> bool: + """Whether a stat finds nothing at `path`. + + A stat needs no file descriptor, so it answers where an open failed for want of one. Unlike `_exists`, it takes + no other failure for absence. Discovery keeps an absent `/proc`, so an EACCES taken for absence would not be + read again. + """ + try: + path.stat() + except (FileNotFoundError, NotADirectoryError): + return True + except (OSError, ValueError): + return False + + return False + + def _unescape(field: str) -> str: """Decode the octal escapes in a path field of `/proc/self/mountinfo`.""" # The backslash goes last. Undoing it first would decode `\134040` into a space. diff --git a/src/proclimits/_sensor.py b/src/proclimits/_sensor.py index e80f331..336cbcb 100644 --- a/src/proclimits/_sensor.py +++ b/src/proclimits/_sensor.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from . import _backend -from ._types import Notice, NoticeCode, RawMemory, Source +from ._types import Notice, NoticeCode, RawCpu, RawMemory, Source _SHORTEST_WINDOW_SECONDS = 0.01 """Below this a rate says more about the counter's own resolution than about the load.""" @@ -96,6 +96,9 @@ class Description: A limit of `None` next to no notice about it means the mechanism was there and nothing limited this process in a way that kills it. Only hard limits are read. + + The sources are located after the readings, so beside a `*_METRICS_UNREADABLE` notice they can show what a + later lookup found. """ memory_budget: MemoryBudget | None @@ -113,18 +116,21 @@ class Description: """ memory_source: Source | None - """Where the memory limit and usage are read from. `None` where no mechanism is present.""" + """Where the memory limit and usage are read from. `None` where no mechanism is present, or where it + cannot be located right now.""" cpu_quota_source: Source | None - """Where the CPU bandwidth quota is read from. `None` where no mechanism is present.""" + """Where the CPU bandwidth quota is read from. `None` where no mechanism is present, or where it cannot be + located right now.""" cpu_set_source: Source | None - """Where the set of allowed cores is read from. `None` where no mechanism is present.""" + """Where the set of allowed cores is read from. `None` where no mechanism is present, or where it cannot + be located right now.""" cpu_usage_source: Source | None """Where the consumed CPU time is read from, as a mechanism and not as a level. Which level is counted is under `cpu_usage`, and `cpu_rate_level` names where a rate is measured. `None` where no mechanism is - present.""" + present, or where it cannot be located right now.""" raw_memory_limit: int | None """The tightest memory limit before filtering. Sentinels included.""" @@ -327,14 +333,13 @@ def describe() -> Description: """Explain how the sensor arrives at its readings. Every reading is taken again, so a limit changed between two calls shows up here. Where the readings come - from was discovered once - `clear_cache()` is what forgets that. + from was discovered once - `clear_cache()` is what forgets that. A lookup that cannot read `/proc` is not + kept. Returns: The readings, the source of each metric, the raw values before filtering, the machine facts, and a notice for whatever is worth knowing about them. """ - sources = _backend.sources() - # One read of each filtering fact, here and in the report, so what decided a rejection is the number # reported beside it. machine_memory = get_machine_memory_bytes() @@ -343,6 +348,9 @@ def describe() -> Description: memory = _evaluate_memory(ceiling) cpu = _evaluate_cpu_limit(machine_cpus) + # After the readings: a lookup that succeeded is kept, so a limit read above has its source here. + sources = _backend.sources() + return Description( memory_budget=memory.effective, cpu_limit=cpu.effective, @@ -370,8 +378,9 @@ def describe() -> Description: def clear_cache() -> None: """Forget the discovered metric sources. - Where a mechanism has anything to discover, it is discovered once and kept. Call this after the process - was moved into another group, and the next reading locates the sources again. + Where a mechanism has anything to discover, it is discovered once and kept. A lookup that cannot read + `/proc` is not kept. Call this after the process was moved into another group, and the next reading + locates the sources again. A child of `fork` clears it on its own, so a pre-fork server whose supervisor puts each worker into a group of its own needs no call here. @@ -590,15 +599,7 @@ def _memory_rejection(raw: RawMemory, ceiling: int | None) -> Notice | None: if raw.limit is None: # No limit and no mechanism read the same way from the outside, and only one of them is a fact about # this machine. - return ( - Notice( - code=NoticeCode.MEMORY_METRICS_UNAVAILABLE, - message='No mechanism here carries a memory limit, so nothing was read. A Linux machine with ' - 'no cgroup filesystem mounted looks like this, and so does a platform with no backend here.', - ) - if _backend.sources().memory is None - else None - ) + return _missing_memory_mechanism(raw) if ceiling is None: return Notice( @@ -691,15 +692,9 @@ def _evaluate_cpu_limit(machine_cpus: int | None) -> _CpuEvaluation: notices = [] candidates = [] - if not restrictions and _no_cpu_mechanism(): + if not restrictions and (missing := _missing_cpu_mechanism(raw)) is not None: # As for memory: "nothing limits the CPU here" and "nothing here can say" are different answers. - notices.append( - Notice( - code=NoticeCode.CPU_METRICS_UNAVAILABLE, - message='No mechanism here carries a CPU limit, so nothing was read. A Linux machine with no ' - 'cgroup filesystem mounted looks like this, and so does a platform with no backend here.', - ) - ) + notices.append(missing) for restriction in restrictions: # A reading in cores is trusted where the machine core count is unknown: cores have no sentinel, so @@ -751,11 +746,64 @@ def _no_cpu_limit(notice: Notice) -> _CpuEvaluation: ) -def _no_cpu_mechanism() -> bool: - """Whether no mechanism here could carry a CPU limit of either kind.""" +def _missing_memory_mechanism(raw: RawMemory) -> Notice | None: + """Say why no source of a memory limit is here: none is present, or none can be located right now. + + `None` where one is. + """ + # A lookup that failed leaves no source either, and says nothing about this machine. The reading's own lookup + # decides first, so a lookup that succeeds after it cannot hide the failure. + if raw.unlocated_file is not None: + return _unlocated_notice(NoticeCode.MEMORY_METRICS_UNREADABLE, 'memory', raw.unlocated_file) + + # The reading's lookup succeeded or found no `/proc`, and either answer is kept, so this one is usually the + # same. Another thread can clear the cache in between, and this lookup then looks again and can fail. + sources = _backend.sources() + + if sources.unlocated_file is not None: + return _unlocated_notice(NoticeCode.MEMORY_METRICS_UNREADABLE, 'memory', sources.unlocated_file) + + if sources.memory is not None: + return None + + return Notice( + code=NoticeCode.MEMORY_METRICS_UNAVAILABLE, + message='No mechanism here carries a memory limit, so nothing was read. A Linux machine with ' + 'no cgroup filesystem mounted looks like this, and so does a platform with no backend here.', + ) + + +def _missing_cpu_mechanism(raw: RawCpu) -> Notice | None: + """Say why no source of a CPU limit of either kind is here: none is present, or none can be located right now. + + `None` where one is. + """ + # As for memory: the reading's own lookup decides first, and this one can still fail after a cleared cache. + if raw.unlocated_file is not None: + return _unlocated_notice(NoticeCode.CPU_METRICS_UNREADABLE, 'CPU', raw.unlocated_file) + sources = _backend.sources() - return sources.cpu_quota is None and sources.cpu_set is None + if sources.unlocated_file is not None: + return _unlocated_notice(NoticeCode.CPU_METRICS_UNREADABLE, 'CPU', sources.unlocated_file) + + if sources.cpu_quota is not None or sources.cpu_set is not None: + return None + + return Notice( + code=NoticeCode.CPU_METRICS_UNAVAILABLE, + message='No mechanism here carries a CPU limit, so nothing was read. A Linux machine with no ' + 'cgroup filesystem mounted looks like this, and so does a platform with no backend here.', + ) + + +def _unlocated_notice(code: NoticeCode, metric: str, unlocated_file: str) -> Notice: + """Say that where a limit would be read from cannot be located, because a file cannot be read.""" + return Notice( + code=code, + message=f'Where a {metric} limit would be read from cannot be located, because {unlocated_file} cannot ' + 'be read. Nothing is reported, and the next reading tries again.', + ) def get_machine_memory_bytes() -> int | None: diff --git a/src/proclimits/_types.py b/src/proclimits/_types.py index 1dfdb3b..39f7dc8 100644 --- a/src/proclimits/_types.py +++ b/src/proclimits/_types.py @@ -65,6 +65,20 @@ class NoticeCode(str, Enum): CPU_METRICS_UNAVAILABLE = 'cpu-metrics-unavailable' """No mechanism here is present to carry a CPU limit: an unsupported platform, or none is mounted.""" + MEMORY_METRICS_UNREADABLE = 'memory-metrics-unreadable' + """Where a memory limit would be read from cannot be located, because a `/proc` file cannot be read. + + Nothing is reported. A read that failed, so the next reading tries again. The message names the file and + the reason. + """ + + CPU_METRICS_UNREADABLE = 'cpu-metrics-unreadable' + """Where a CPU limit would be read from cannot be located, because a `/proc` file cannot be read. + + Nothing is reported. A read that failed, so the next reading tries again. The message names the file and + the reason. + """ + MEMORY_LIMIT_UNREADABLE = 'memory-limit-unreadable' """A level holds a memory limit that says nothing usable, so what it enforces is unknown.""" @@ -145,10 +159,19 @@ class Sources: cpu_usage: Source | None """Carries the consumed CPU time, which a mechanism may count somewhere other than where it limits.""" + unlocated_file: str | None + """The file that kept the sources from being located, and why, as ` ()`. `None` where none did. + + Set, every source is `None`, and the next lookup tries again. + """ + @dataclass(frozen=True) class RawMemory: - """The memory limit and usage as the mechanism spells them.""" + """The memory limit and usage as the mechanism spells them. + + Where `unlocated_file` is set, no level was read and every other field is `None`. + """ limit: int | None """The tightest limit along the chain, in bytes. `None` when no level holds one. @@ -200,6 +223,13 @@ class RawMemory: no level to name. """ + unlocated_file: str | None + """The `/proc` file this reading could not read to locate its controller, and why, as ` ()`. + + `None` where the lookup succeeded or found no `/proc`. A job object is not located through `/proc`, so it is + always `None` there. + """ + @dataclass(frozen=True) class RawCpuQuota: @@ -247,7 +277,8 @@ class RawCpuSet: class RawCpu: """Everything the CPU controls say, read in one go. - A level that says nothing usable concerns both restrictions, so they are read together. + A level that says nothing usable concerns both restrictions, so they are read together. Where + `unlocated_file` is set, no level was read and every other field is `None`. """ quota: RawCpuQuota | None @@ -270,3 +301,10 @@ class RawCpu: mechanisms that express a limit that way ever set this - a cgroup states cores outright, so it is always `None` there. """ + + unlocated_file: str | None + """The `/proc` file this reading could not read to locate its controllers, and why, as ` ()`. + + `None` where the lookup succeeded or found no `/proc`. A job object is not located through `/proc`, so it is + always `None` there. + """ diff --git a/src/proclimits/_windows.py b/src/proclimits/_windows.py index bd9299f..962b9fd 100644 --- a/src/proclimits/_windows.py +++ b/src/proclimits/_windows.py @@ -293,6 +293,7 @@ class _SystemInfo(ctypes.Structure): limit_level=None, unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) """What is reported where no job limits the memory.""" @@ -304,6 +305,7 @@ class _SystemInfo(ctypes.Structure): limit_level=None, unreadable_level=_LEVEL, usage_unreadable_level=None, + unlocated_file=None, ) """What is reported where this process is in a job that will not say what it enforces.""" @@ -337,6 +339,7 @@ def read_memory() -> RawMemory: limit_level=_LEVEL, unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) if job_limit is None: @@ -358,6 +361,7 @@ def read_cpu() -> RawCpu: cpu_set=None, unreadable_level=None if error == _ERROR_NO_JOB else _LEVEL, unconvertible_level=None, + unlocated_file=None, ) # One unreadable answer drops both restrictions, because the unread one may be the tighter. No machine @@ -365,7 +369,7 @@ def read_cpu() -> RawCpu: # were measured to answer or fail together. control, _ = _query(_JOB_CPU_RATE_CONTROL, _RateControl) if control is None: - return RawCpu(quota=None, cpu_set=None, unreadable_level=_LEVEL, unconvertible_level=None) + return RawCpu(quota=None, cpu_set=None, unreadable_level=_LEVEL, unconvertible_level=None, unlocated_file=None) rate = _rate_cap(control) machine_cores = machine_cpu_count() @@ -373,7 +377,7 @@ def read_cpu() -> RawCpu: if rate is not None and machine_cores is None: # A rate is a share of the machine, so without the size of the machine it says nothing about cores. # The affinity mask goes with it: the share that cannot be sized may be the tighter of the two. - return RawCpu(quota=None, cpu_set=None, unreadable_level=None, unconvertible_level=_LEVEL) + return RawCpu(quota=None, cpu_set=None, unreadable_level=None, unconvertible_level=_LEVEL, unlocated_file=None) quota = ( RawCpuQuota(cores=rate * machine_cores, limit_level=_LEVEL, usage_level=_LEVEL) @@ -391,11 +395,13 @@ def read_cpu() -> RawCpu: if allowed is None: # The job restricts the cores and will not say which, so what it allows is unknown rather than # unrestricted. - return RawCpu(quota=None, cpu_set=None, unreadable_level=_LEVEL, unconvertible_level=None) + return RawCpu( + quota=None, cpu_set=None, unreadable_level=_LEVEL, unconvertible_level=None, unlocated_file=None + ) cpu_set = RawCpuSet(cores=allowed, limit_level=_LEVEL, usage_level=_LEVEL) - return RawCpu(quota=quota, cpu_set=cpu_set, unreadable_level=None, unconvertible_level=None) + return RawCpu(quota=quota, cpu_set=cpu_set, unreadable_level=None, unconvertible_level=None, unlocated_file=None) def read_cpu_usage(level: str | None = None) -> float | None: @@ -430,7 +436,7 @@ def sources() -> Sources: """ source = Source(interface=Interface.WINDOWS_JOB_OBJECT, levels=(_LEVEL,) if _in_job() else ()) - return Sources(memory=source, cpu_quota=source, cpu_set=source, cpu_usage=source) + return Sources(memory=source, cpu_quota=source, cpu_set=source, cpu_usage=source, unlocated_file=None) def mechanism_notices() -> tuple[Notice, ...]: @@ -574,6 +580,7 @@ def _job_memory(limit: int) -> RawMemory: # Nothing charged takes two unrelated APIs refusing at once, and has never been seen on a machine. # A job-wide limit does have a usage figure, so this is a guard rather than a path. usage_unreadable_level=_LEVEL if charged is None else None, + unlocated_file=None, ) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index c719991..d3f4a9a 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +from pathlib import Path from typing import TYPE_CHECKING import pytest @@ -8,7 +9,6 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterator - from pathlib import Path V2_MOUNTINFO = '25 30 0:22 / {root} rw,nosuid,nodev,noexec,relatime shared:4 - cgroup2 cgroup2 rw,nsdelegate' """A single unified hierarchy exposed from its top, which is what a container runtime sets up.""" @@ -53,6 +53,22 @@ """The two interfaces name different cgroups for the same process, which is what makes the layout dangerous.""" +def fail_to_read(monkeypatch: pytest.MonkeyPatch, failing: Path, error: OSError | ValueError, *, times: int) -> None: + """Make reading one file raise `error` the first `times` times, and read it as usual after that. + + The error names no file, as one raised by a read after the file was opened does not. + """ + read_text = Path.read_text + failures = [error] * times + + def read_or_fail(path: Path, encoding: str | None = None, errors: str | None = None) -> str: + if path == failing and failures: + raise failures.pop() + return read_text(path, encoding=encoding, errors=errors) + + monkeypatch.setattr(Path, 'read_text', read_or_fail) + + @pytest.fixture(autouse=True) def _isolated_module_state() -> Iterator[None]: """Reset the process-wide discovery cache, so that nothing leaks between tests.""" diff --git a/tests/unit/test_cgroup.py b/tests/unit/test_cgroup.py index e428d4e..59f619c 100644 --- a/tests/unit/test_cgroup.py +++ b/tests/unit/test_cgroup.py @@ -1,5 +1,7 @@ from __future__ import annotations +import errno +import os from pathlib import Path from typing import TYPE_CHECKING @@ -17,11 +19,15 @@ V1_SELF_CGROUP, V2_MOUNTINFO, V2_SELF_CGROUP, + fail_to_read, ) if TYPE_CHECKING: from collections.abc import Callable +MEMORY_FILES = {'memory.max': '1000\n', 'memory.current': '100\n', 'memory.stat': 'inactive_file 0\n'} +"""A memory limit at the top of the mount, so a lookup that succeeds locates the memory controller.""" + def test_read_hierarchies_v2(fake_cgroup: Callable[..., Path]) -> None: """Finds the unified hierarchy and the cgroup this process belongs to in it.""" @@ -180,6 +186,7 @@ def test_read_hierarchies_no_unified_when_no_mount_covers(fake_cgroup: Callable[ limit_level=None, unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -207,6 +214,7 @@ def test_read_hierarchies_v1_answers_when_no_cgroup2_mount_covers(fake_cgroup: C limit_level=str(root / 'memory'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -275,6 +283,213 @@ def test_read_own_paths( assert controllers == expected_controllers +def count_discoveries(monkeypatch: pytest.MonkeyPatch) -> list[None]: + """Count the walks over `/proc`, one entry for each lookup that ran discovery rather than reusing it.""" + read_hierarchies = cgroup._read_hierarchies + discoveries: list[None] = [] + + def counted() -> cgroup._Hierarchies: + discoveries.append(None) + return read_hierarchies() + + monkeypatch.setattr(cgroup, '_read_hierarchies', counted) + + return discoveries + + +@pytest.mark.parametrize( + ('error', 'reason'), + [ + pytest.param(errno.EMFILE, 'EMFILE', id='the process has no descriptor left'), + pytest.param(errno.ENFILE, 'ENFILE', id='the system has no descriptor left'), + pytest.param(errno.ENOMEM, 'ENOMEM', id='the kernel has no memory left'), + pytest.param(errno.EACCES, 'EACCES', id='the file may not be read'), + ], +) +@pytest.mark.parametrize('failing', ['_PROC_SELF_CGROUP', '_PROC_SELF_MOUNTINFO']) +def test_locate_controllers_retries_a_failed_proc_read( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, + failing: str, + error: int, + reason: str, +) -> None: + """Names the `/proc` file that failed and why, keeps nothing, and locates on the next lookup.""" + fake_cgroup(mountinfo=V2_MOUNTINFO, self_cgroup=V2_SELF_CGROUP.format(path='/'), files=MEMORY_FILES) + path = getattr(cgroup, failing) + fail_to_read(monkeypatch, path, OSError(error, os.strerror(error)), times=1) + discoveries = count_discoveries(monkeypatch) + + assert cgroup.locate_controllers() == cgroup.Controllers( + interface=None, + memory=None, + cpu_quota=None, + cpu_usage=None, + cpu_set=None, + unlocated_file=f'{path} ({reason})', + ) + located = cgroup.locate_controllers() + assert located.memory is not None + # The success is kept, so a third lookup walks nothing. + assert cgroup.locate_controllers() == located + assert len(discoveries) == 2 + + +@pytest.mark.parametrize( + 'error', + [ + pytest.param(FileNotFoundError(errno.ENOENT, 'No such file or directory'), id='no such file'), + pytest.param(NotADirectoryError(errno.ENOTDIR, 'Not a directory'), id='a file where a directory belongs'), + ], +) +def test_locate_controllers_keeps_an_absent_proc( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, + error: OSError, +) -> None: + """Keeps a `/proc` that is not there, which lasts as long as the process, and names no file.""" + fake_cgroup(mountinfo=V2_MOUNTINFO, self_cgroup=V2_SELF_CGROUP.format(path='/'), files=MEMORY_FILES) + fail_to_read(monkeypatch, cgroup._PROC_SELF_CGROUP, error, times=3) + discoveries = count_discoveries(monkeypatch) + + located_nothing = cgroup.Controllers( + interface=None, memory=None, cpu_quota=None, cpu_usage=None, cpu_set=None, unlocated_file=None + ) + assert [cgroup.locate_controllers() for _ in range(3)] == [located_nothing] * 3 + assert len(discoveries) == 1 + + +@pytest.mark.usefixtures('_no_cgroup') +def test_locate_controllers_keeps_an_absent_proc_out_of_descriptors(monkeypatch: pytest.MonkeyPatch) -> None: + """Keeps a `/proc` file that is not there where opening it fails for want of file descriptors.""" + # Out of descriptors, opening a file that is not there fails with EMFILE rather than ENOENT. + emfile = OSError(errno.EMFILE, os.strerror(errno.EMFILE)) + fail_to_read(monkeypatch, cgroup._PROC_SELF_CGROUP, emfile, times=3) + discoveries = count_discoveries(monkeypatch) + + located_nothing = cgroup.Controllers( + interface=None, memory=None, cpu_quota=None, cpu_usage=None, cpu_set=None, unlocated_file=None + ) + assert [cgroup.locate_controllers() for _ in range(3)] == [located_nothing] * 3 + assert len(discoveries) == 1 + + +@pytest.mark.usefixtures('_no_cgroup') +def test_locate_controllers_retries_where_a_stat_cannot_tell_absence(monkeypatch: pytest.MonkeyPatch) -> None: + """Keeps nothing where the open and the stat of a `/proc` file both fail, and locates on the next lookup.""" + path = cgroup._PROC_SELF_CGROUP + stat = Path.stat + + def refuse_stat(self: Path, *, follow_symlinks: bool = True) -> os.stat_result: + if self == path: + raise PermissionError(errno.EACCES, os.strerror(errno.EACCES)) + return stat(self, follow_symlinks=follow_symlinks) + + discoveries = count_discoveries(monkeypatch) + + # A refused stat says nothing about whether the file is there. Taken for absence, it would be kept. + with monkeypatch.context() as patch: + fail_to_read(patch, path, OSError(errno.EMFILE, os.strerror(errno.EMFILE)), times=1) + patch.setattr(Path, 'stat', refuse_stat) + failed = cgroup.locate_controllers() + + assert failed.unlocated_file == f'{path} (EMFILE)' + assert cgroup.locate_controllers().unlocated_file is None + assert len(discoveries) == 2 + + +def test_locate_controllers_retries_a_value_error( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Keeps no `ValueError`: content that cannot be read now may be read on the next lookup.""" + fake_cgroup(mountinfo=V2_MOUNTINFO, self_cgroup=V2_SELF_CGROUP.format(path='/'), files=MEMORY_FILES) + path = cgroup._PROC_SELF_CGROUP + fail_to_read(monkeypatch, path, ValueError('unreadable content'), times=3) + discoveries = count_discoveries(monkeypatch) + + lookups = [cgroup.locate_controllers() for _ in range(3)] + + assert [lookup.unlocated_file for lookup in lookups] == [f'{path} (ValueError)'] * 3 + assert len(discoveries) == 3 + + +def test_locate_controllers_out_of_descriptors(fake_cgroup: Callable[..., Path]) -> None: + """Names the file a real descriptor shortage failed, and locates once descriptors are free again.""" + # A descriptor limit is set through `resource`, which is POSIX. + resource = pytest.importorskip('resource') + fake_cgroup(mountinfo=V2_MOUNTINFO, self_cgroup=V2_SELF_CGROUP.format(path='/'), files=MEMORY_FILES) + + # Every descriptor below the lowest free one is taken, so a limit there leaves no room for another. + lowest_free = os.open(os.devnull, os.O_RDONLY) + os.close(lowest_free) + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + try: + resource.setrlimit(resource.RLIMIT_NOFILE, (lowest_free, hard)) + failed = cgroup.locate_controllers() + finally: + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + + assert failed.unlocated_file == f'{cgroup._PROC_SELF_CGROUP} (EMFILE)' + assert cgroup.locate_controllers().memory is not None + + +def test_read_memory_names_a_failed_lookup( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Names the `/proc` file its own lookup could not read, and names none once a lookup succeeds.""" + root = fake_cgroup(mountinfo=V2_MOUNTINFO, self_cgroup=V2_SELF_CGROUP.format(path='/'), files=MEMORY_FILES) + path = cgroup._PROC_SELF_CGROUP + fail_to_read(monkeypatch, path, OSError(errno.EMFILE, os.strerror(errno.EMFILE)), times=1) + + assert cgroup.read_memory() == cgroup.RawMemory( + limit=None, + used=None, + available=None, + unflushed_cache=None, + limit_level=None, + unreadable_level=None, + usage_unreadable_level=None, + unlocated_file=f'{path} (EMFILE)', + ) + assert cgroup.read_memory() == cgroup.RawMemory( + limit=1000, + used=100, + available=900, + unflushed_cache=None, + limit_level=str(root), + unreadable_level=None, + usage_unreadable_level=None, + unlocated_file=None, + ) + + +def test_read_cpu_names_a_failed_lookup( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Names the `/proc` file its own lookup could not read, and names none once a lookup succeeds.""" + root = fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + path = cgroup._PROC_SELF_CGROUP + fail_to_read(monkeypatch, path, OSError(errno.EMFILE, os.strerror(errno.EMFILE)), times=1) + + assert cgroup.read_cpu() == cgroup.RawCpu( + quota=None, cpu_set=None, unreadable_level=None, unconvertible_level=None, unlocated_file=f'{path} (EMFILE)' + ) + assert cgroup.read_cpu() == cgroup.RawCpu( + quota=cgroup.RawCpuQuota(cores=2.0, limit_level=str(root), usage_level=str(root)), + cpu_set=None, + unreadable_level=None, + unconvertible_level=None, + unlocated_file=None, + ) + + @pytest.mark.parametrize( ('mount_root', 'cgroup_path', 'expected'), [ @@ -408,6 +623,7 @@ def test_read_memory_tightest_level(fake_cgroup: Callable[..., Path]) -> None: limit_level=str(root / 'kubepods' / 'pod' / 'container'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -439,6 +655,7 @@ def test_read_memory_when_the_tightest_limit_is_not_the_closest(fake_cgroup: Cal limit_level=str(root / 'pod' / 'container'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -468,6 +685,7 @@ def test_read_memory_tighter_ancestor(fake_cgroup: Callable[..., Path]) -> None: limit_level=str(root / 'kubepods'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -497,6 +715,7 @@ def test_read_memory_smallest_distance(fake_cgroup: Callable[..., Path]) -> None limit_level=str(root / 'kubepods' / 'pod'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -524,6 +743,7 @@ def test_read_memory_partial_level(fake_cgroup: Callable[..., Path]) -> None: limit_level=str(root / 'kubepods' / 'pod' / 'container'), unreadable_level=None, usage_unreadable_level=str(root / 'kubepods' / 'pod' / 'container'), + unlocated_file=None, ) @@ -580,6 +800,7 @@ def test_read_memory_usage_above_the_limit(fake_cgroup: Callable[..., Path]) -> limit_level=str(root), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -610,6 +831,7 @@ def test_read_memory_ancestor_without_usage_files(fake_cgroup: Callable[..., Pat limit_level=str(root / 'pod'), unreadable_level=None, usage_unreadable_level=str(root / 'pod'), + unlocated_file=None, ) @@ -629,6 +851,7 @@ def test_read_memory_no_used(fake_cgroup: Callable[..., Path]) -> None: limit_level=str(root), unreadable_level=None, usage_unreadable_level=str(root), + unlocated_file=None, ) @@ -654,6 +877,7 @@ def test_read_memory_missing_files(fake_cgroup: Callable[..., Path]) -> None: limit_level=str(root / 'pod'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -734,7 +958,7 @@ def test_read_cpu_quota( """Reads the CPU bandwidth quota under both cgroup interfaces.""" fake_cgroup(mountinfo=mountinfo, self_cgroup=self_cgroup, files=files) - quota = cgroup.read_cpu_quota() + quota = cgroup.read_cpu_quota(cgroup.locate_controllers()) assert (quota.cores if quota is not None else None) == expected @@ -782,7 +1006,7 @@ def test_read_cpu_set_size( """Counts the cores of a CPU set spelled as a mix of ranges and single numbers.""" fake_cgroup(mountinfo=mountinfo, self_cgroup=self_cgroup, files=files) - cpu_set = cgroup.read_cpu_set_size() + cpu_set = cgroup.read_cpu_set_size(cgroup.locate_controllers()) assert (cpu_set.cores if cpu_set is not None else None) == expected @@ -836,14 +1060,14 @@ def test_read_cpu_set_size_file_disappears(fake_cgroup: Callable[..., Path]) -> self_cgroup=V2_SELF_CGROUP.format(path='/'), files={'cpuset.cpus.effective': '0-1\n'}, ) - cpu_set = cgroup.read_cpu_set_size() + cpu_set = cgroup.read_cpu_set_size(cgroup.locate_controllers()) assert cpu_set is not None assert cpu_set.cores == 2 (root / 'cpuset.cpus.effective').unlink() - assert cgroup.read_cpu_set_size() is None + assert cgroup.read_cpu_set_size(cgroup.locate_controllers()) is None def test_read_cpu_set_size_unreadable_file(fake_cgroup: Callable[..., Path]) -> None: @@ -858,7 +1082,7 @@ def test_read_cpu_set_size_unreadable_file(fake_cgroup: Callable[..., Path]) -> (root / 'cpuset.cpus.effective').mkdir() assert cgroup.read_cpu() == cgroup.RawCpu( - quota=None, cpu_set=None, unreadable_level=str(root), unconvertible_level=None + quota=None, cpu_set=None, unreadable_level=str(root), unconvertible_level=None, unlocated_file=None ) @@ -939,7 +1163,7 @@ def test_cpu_usage_dir_rejects_a_group_of_the_same_name(fake_cgroup: Callable[.. }, ) - quota = cgroup.read_cpu_quota() + quota = cgroup.read_cpu_quota(cgroup.locate_controllers()) assert quota is not None assert quota.cores == 2.0 @@ -970,6 +1194,7 @@ def test_read_memory_unreadable_limit(fake_cgroup: Callable[..., Path]) -> None: limit_level=None, unreadable_level=str(root / 'pod' / 'container'), usage_unreadable_level=None, + unlocated_file=None, ) @@ -998,6 +1223,7 @@ def test_read_memory_looser_level_without_usage(fake_cgroup: Callable[..., Path] limit_level=str(root / 'pod' / 'container'), unreadable_level=None, usage_unreadable_level=str(root / 'pod'), + unlocated_file=None, ) @@ -1053,6 +1279,7 @@ def test_read_cpu_quota_unreadable_level(fake_cgroup: Callable[..., Path]) -> No cpu_set=None, unreadable_level=str(root / 'pod' / 'container'), unconvertible_level=None, + unlocated_file=None, ) @@ -1068,7 +1295,7 @@ def test_read_cpu_set_size_inherited_under_v1(fake_cgroup: Callable[..., Path]) }, ) - cpu_set = cgroup.read_cpu_set_size() + cpu_set = cgroup.read_cpu_set_size(cgroup.locate_controllers()) assert cpu_set is not None assert cpu_set.cores == 2 @@ -1083,7 +1310,7 @@ def test_read_cpu_set_size_without_a_counter_of_its_own(fake_cgroup: Callable[.. files={'cpuset/limited/cpuset.cpus': '0-1\n', 'cpuacct/cpuacct.usage': '5000000000\n'}, ) - cpu_set = cgroup.read_cpu_set_size() + cpu_set = cgroup.read_cpu_set_size(cgroup.locate_controllers()) assert cpu_set is not None assert cpu_set.cores == 2 @@ -1102,7 +1329,7 @@ def test_read_cpu_set_size_from_an_ancestor(fake_cgroup: Callable[..., Path]) -> }, ) - cpu_set = cgroup.read_cpu_set_size() + cpu_set = cgroup.read_cpu_set_size(cgroup.locate_controllers()) assert cpu_set is not None assert cpu_set.cores == 2 @@ -1120,7 +1347,7 @@ def test_read_cpu_set_size_without_any_accounting_hierarchy(fake_cgroup: Callabl files={'cpuset/limited/cpuset.cpus': '0-1\n'}, ) - cpu_set = cgroup.read_cpu_set_size() + cpu_set = cgroup.read_cpu_set_size(cgroup.locate_controllers()) assert cpu_set is not None assert cpu_set.cores == 2 @@ -1138,7 +1365,7 @@ def test_read_cpu_set_size_across_split_hierarchies(fake_cgroup: Callable[..., P }, ) - cpu_set = cgroup.read_cpu_set_size() + cpu_set = cgroup.read_cpu_set_size(cgroup.locate_controllers()) assert cpu_set is not None assert cpu_set.limit_level == str(root / 'cpuset' / 'limited') @@ -1169,6 +1396,7 @@ def test_locate_controllers_hybrid(fake_cgroup: Callable[..., Path]) -> None: limit_level=str(root / 'memory'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1183,9 +1411,14 @@ def test_no_cgroups() -> None: limit_level=None, unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, + ) + # An absent `/proc` is a fact about the system rather than a failed lookup, so no file is named. + assert cgroup.read_cpu() == cgroup.RawCpu( + quota=None, cpu_set=None, unreadable_level=None, unconvertible_level=None, unlocated_file=None ) - assert cgroup.read_cpu_quota() is None - assert cgroup.read_cpu_set_size() is None + assert cgroup.read_cpu_quota(cgroup.locate_controllers()) is None + assert cgroup.read_cpu_set_size(cgroup.locate_controllers()) is None assert cgroup.read_cpu_usage() is None @@ -1222,6 +1455,7 @@ def test_locate_controllers_with_unreadable_directories( limit_level=None, unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1251,7 +1485,7 @@ def test_read_cpu_quota_tightest_level(fake_cgroup: Callable[..., Path]) -> None ) # The quota binds on the pod, so a rate has to be measured there rather than in the container. - assert cgroup.read_cpu_quota() == cgroup.RawCpuQuota( + assert cgroup.read_cpu_quota(cgroup.locate_controllers()) == cgroup.RawCpuQuota( cores=1.5, limit_level=str(root / 'pod'), usage_level=str(root / 'pod'), @@ -1271,7 +1505,7 @@ def test_read_cpu_quota_on_the_own_group(fake_cgroup: Callable[..., Path]) -> No }, ) - assert cgroup.read_cpu_quota() == cgroup.RawCpuQuota( + assert cgroup.read_cpu_quota(cgroup.locate_controllers()) == cgroup.RawCpuQuota( cores=1.5, limit_level=str(root / 'pod' / 'container'), usage_level=str(root / 'pod' / 'container'), @@ -1322,7 +1556,7 @@ def test_read_cpu_set_size_reads_the_closest_level(fake_cgroup: Callable[..., Pa }, ) - cpu_set = cgroup.read_cpu_set_size() + cpu_set = cgroup.read_cpu_set_size(cgroup.locate_controllers()) assert cpu_set is not None assert cpu_set.cores == 2 @@ -1367,6 +1601,7 @@ def test_read_memory_limit_of_zero(fake_cgroup: Callable[..., Path]) -> None: limit_level=str(root / 'pod' / 'container'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1393,6 +1628,7 @@ def test_read_memory_usage_below_the_cache(fake_cgroup: Callable[..., Path]) -> limit_level=str(root), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1458,6 +1694,7 @@ def test_read_hierarchies_skips_a_foreign_v1_mount(fake_cgroup: Callable[..., Pa limit_level=str(root / 'ours' / 'mine'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1481,6 +1718,7 @@ def test_read_memory_v1_uses_the_hierarchical_key(fake_cgroup: Callable[..., Pat limit_level=str(root / 'memory'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1499,7 +1737,7 @@ def test_read_cpu_quota_across_split_hierarchies(fake_cgroup: Callable[..., Path }, ) - quota = cgroup.read_cpu_quota() + quota = cgroup.read_cpu_quota(cgroup.locate_controllers()) assert quota is not None assert quota.cores == 0.5 @@ -1522,7 +1760,7 @@ def test_read_cpu_quota_without_accounting(fake_cgroup: Callable[..., Path]) -> }, ) - quota = cgroup.read_cpu_quota() + quota = cgroup.read_cpu_quota(cgroup.locate_controllers()) assert quota is not None assert quota.cores == 0.5 @@ -1561,12 +1799,12 @@ def test_read_cpu_quota_appearing_after_discovery(fake_cgroup: Callable[..., Pat }, ) - assert cgroup.read_cpu_quota() is None + assert cgroup.read_cpu_quota(cgroup.locate_controllers()) is None # The scope of this process is given a quota of its own, below where the chain would have been cut. (root / 'slice' / 'own' / 'cpu.max').write_text('50000 100000\n') - quota = cgroup.read_cpu_quota() + quota = cgroup.read_cpu_quota(cgroup.locate_controllers()) assert quota is not None assert quota.cores == 0.5 @@ -1595,6 +1833,7 @@ def test_read_memory_dirty_cache_is_not_credited(fake_cgroup: Callable[..., Path limit_level=str(root), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1622,6 +1861,7 @@ def test_read_memory_dirty_cache_is_not_credited_v1(fake_cgroup: Callable[..., P limit_level=str(root / 'memory'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1646,6 +1886,7 @@ def test_read_memory_more_waiting_than_inactive_cache(fake_cgroup: Callable[..., limit_level=str(root), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1676,6 +1917,7 @@ def test_read_memory_without_the_dirty_keys(fake_cgroup: Callable[..., Path]) -> limit_level=str(root / 'memory'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1699,6 +1941,7 @@ def test_read_memory_dirty_keys_without_the_inactive_cache(fake_cgroup: Callable limit_level=str(root), unreadable_level=None, usage_unreadable_level=str(root), + unlocated_file=None, ) @@ -1729,6 +1972,7 @@ def test_read_memory_reports_the_cache_of_the_deciding_level(fake_cgroup: Callab limit_level=str(root / 'kubepods' / 'pod'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1760,6 +2004,7 @@ def test_read_memory_reports_the_cache_of_the_closest_level_on_a_tie(fake_cgroup limit_level=str(root / 'kubepods' / 'pod'), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1826,6 +2071,7 @@ def test_read_memory_with_the_dirty_keys_at_zero(fake_cgroup: Callable[..., Path limit_level=str(root), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1850,6 +2096,7 @@ def test_read_memory_unparsable_dirty_key(fake_cgroup: Callable[..., Path]) -> N limit_level=str(root), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1886,6 +2133,7 @@ def test_read_memory_negative_dirty_key(fake_cgroup: Callable[..., Path]) -> Non limit_level=str(root), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) @@ -1910,4 +2158,5 @@ def test_read_memory_negative_key_beside_a_positive_one(fake_cgroup: Callable[.. limit_level=str(root), unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) diff --git a/tests/unit/test_sensor.py b/tests/unit/test_sensor.py index eb8d4fa..c606130 100644 --- a/tests/unit/test_sensor.py +++ b/tests/unit/test_sensor.py @@ -1,10 +1,12 @@ from __future__ import annotations import asyncio +import errno import json import os import threading from itertools import count +from pathlib import Path from types import SimpleNamespace from typing import TYPE_CHECKING @@ -21,11 +23,11 @@ V1_SELF_CGROUP, V2_MOUNTINFO, V2_SELF_CGROUP, + fail_to_read, ) if TYPE_CHECKING: from collections.abc import Callable - from pathlib import Path MACHINE_TOTAL_BYTES = 8 * 1024**3 MACHINE_CORES = 8 @@ -201,6 +203,7 @@ def test_get_memory_budget_usage_the_mechanism_does_not_offer(monkeypatch: pytes limit_level='job', unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) monkeypatch.setattr(_backend, 'read_memory', lambda: raw) @@ -218,6 +221,7 @@ def test_get_memory_budget_brings_a_disagreeing_triple_into_range(monkeypatch: p limit_level='job', unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) monkeypatch.setattr(_backend, 'read_memory', lambda: raw) @@ -238,6 +242,7 @@ def test_get_memory_budget_keeps_a_room_the_mechanism_made_smaller(monkeypatch: limit_level='job', unreadable_level=None, usage_unreadable_level=None, + unlocated_file=None, ) monkeypatch.setattr(_backend, 'read_memory', lambda: raw) @@ -395,8 +400,8 @@ def test_get_cpu_limit( if cpu_set_cores is None else _cgroup.RawCpuSet(cores=cpu_set_cores, limit_level=counter, usage_level=counter) ) - monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda: raw_quota) - monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda: raw_set) + monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda _controllers: raw_quota) + monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda _controllers: raw_set) monkeypatch.setattr(_sensor, 'get_machine_cpu_count', lambda: machine_cores) assert proclimits.get_cpu_limit() == expected @@ -407,9 +412,9 @@ def test_get_cpu_limit_notices(monkeypatch: pytest.MonkeyPatch) -> None: """Explains each reading that covers the machine, so a missing limit is attributable.""" counter = '/sys/fs/cgroup' quota = _cgroup.RawCpuQuota(cores=10.0, limit_level=counter, usage_level=counter) - monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda: quota) + monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda _controllers: quota) cpu_set = _cgroup.RawCpuSet(cores=8, limit_level=counter, usage_level=counter) - monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda: cpu_set) + monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda _controllers: cpu_set) assert proclimits.get_cpu_limit() is None assert notice_codes('cpu') == ('cpu-quota-covers-machine', 'cpu-set-covers-machine') @@ -417,7 +422,9 @@ def test_get_cpu_limit_notices(monkeypatch: pytest.MonkeyPatch) -> None: def test_cpu_limit_from_an_unsizeable_share(monkeypatch: pytest.MonkeyPatch) -> None: """Reports nothing, and names the level, where a limit is a share of a machine of unknown size.""" - raw = _cgroup.RawCpu(quota=None, cpu_set=None, unreadable_level=None, unconvertible_level='job') + raw = _cgroup.RawCpu( + quota=None, cpu_set=None, unreadable_level=None, unconvertible_level='job', unlocated_file=None + ) monkeypatch.setattr(_backend, 'read_cpu', lambda: raw) assert proclimits.get_cpu_limit() is None @@ -433,6 +440,7 @@ def test_an_unsizeable_share_drops_the_other_reading(monkeypatch: pytest.MonkeyP cpu_set=_cgroup.RawCpuSet(cores=2, limit_level=counter, usage_level=counter), unreadable_level=None, unconvertible_level='job', + unlocated_file=None, ) monkeypatch.setattr(_backend, 'read_cpu', lambda: raw) @@ -958,8 +966,8 @@ def test_covers_machine_notice_spells_the_cores(monkeypatch: pytest.MonkeyPatch) """Says "64 allowed cores" in the message a consumer reads, not "64.0" - a set has no fractional size.""" counter = '/sys/fs/cgroup' cpu_set = _cgroup.RawCpuSet(cores=64, limit_level=counter, usage_level=counter) - monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda: None) - monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda: cpu_set) + monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda _controllers: None) + monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda _controllers: cpu_set) (notice,) = [n for n in proclimits.describe().notices if str(n.code) == 'cpu-set-covers-machine'] @@ -1015,6 +1023,228 @@ def test_clear_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, fake_cgrou ) +def test_a_failed_lookup_is_named_and_retried( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Names the `/proc` file while lookups fail, and reports the limit once one succeeds, with no cache cleared.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={ + 'memory.max': '536870912\n', + 'memory.current': '150000000\n', + 'memory.stat': 'inactive_file 50000000\n', + }, + ) + refused = _cgroup._PROC_SELF_CGROUP + read_text = Path.read_text + failing = True + + def read_unless_failing(path: Path, encoding: str | None = None, errors: str | None = None) -> str: + if failing and path == refused: + raise OSError(errno.EMFILE, os.strerror(errno.EMFILE)) + return read_text(path, encoding=encoding, errors=errors) + + monkeypatch.setattr(Path, 'read_text', read_unless_failing) + + description = proclimits.describe() + + assert [(str(notice.code), notice.message) for notice in description.notices] == [ + ( + 'memory-metrics-unreadable', + ( + f'Where a memory limit would be read from cannot be located, because {refused} (EMFILE) cannot be ' + 'read. Nothing is reported, and the next reading tries again.' + ), + ), + ( + 'cpu-metrics-unreadable', + ( + f'Where a CPU limit would be read from cannot be located, because {refused} (EMFILE) cannot be ' + 'read. Nothing is reported, and the next reading tries again.' + ), + ), + ] + assert description.memory_source is None + assert description.cpu_quota_source is None + assert description.cpu_set_source is None + assert description.cpu_usage_source is None + assert proclimits.get_memory_budget() is None + + failing = False + + assert proclimits.get_memory_budget() == proclimits.MemoryBudget( + limit=536870912, used=100000000, available=436870912 + ) + assert proclimits.describe().notices == () + + +def fail_the_first_lookup(monkeypatch: pytest.MonkeyPatch) -> None: + """Make the first read of `/proc/self/cgroup` raise EMFILE, and read it as usual after that.""" + fail_to_read(monkeypatch, _cgroup._PROC_SELF_CGROUP, OSError(errno.EMFILE, os.strerror(errno.EMFILE)), times=1) + + +def test_a_failed_lookup_reports_no_looser_cpu_limit( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reports no CPU limit while a lookup fails, and says why, rather than a looser set the next lookup locates.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpuset.cpus.effective': '0-3\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + refused = _cgroup._PROC_SELF_CGROUP + fail_the_first_lookup(monkeypatch) + + # Evaluated directly: `describe()` reads memory first, and that reading would take the one failure. + evaluation = _sensor._evaluate_cpu_limit(MACHINE_CORES) + + # Only the first `/proc` read fails. It is named, and the set of 4 cores must not stand in for the quota of 2. + assert (evaluation.effective, evaluation.raw_quota, evaluation.raw_set_size) == (None, None, None) + assert [(str(notice.code), notice.message) for notice in evaluation.notices] == [ + ( + 'cpu-metrics-unreadable', + ( + f'Where a CPU limit would be read from cannot be located, because {refused} (EMFILE) cannot be ' + 'read. Nothing is reported, and the next reading tries again.' + ), + ), + ] + + # The same through the public call. No lookup has succeeded yet, so one more failure reaches it. + fail_the_first_lookup(monkeypatch) + + assert proclimits.get_cpu_limit() is None + assert proclimits.get_cpu_limit() == 2.0 + + +def test_a_cache_cleared_during_a_cpu_reading_reports_no_looser_set( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reads the quota and the set from one lookup, so a cache cleared in between cannot drop the quota.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={'cpu.max': '200000 100000\n', 'cpuset.cpus.effective': '0-3\n', 'cpu.stat': 'usage_usec 0\n'}, + ) + locate = _cgroup.locate_controllers + raced: list[None] = [] + + def locate_then_clear() -> _cgroup.Controllers: + controllers = locate() + if not raced: + # Another thread forgets the discovery right after the first lookup, and the next `/proc` read fails. + raced.append(None) + _cgroup.clear_cache() + fail_the_first_lookup(monkeypatch) + return controllers + + monkeypatch.setattr(_cgroup, 'locate_controllers', locate_then_clear) + + evaluation = _sensor._evaluate_cpu_limit(MACHINE_CORES) + + # A second lookup inside the reading would fail here, lose the quota of 2 and report the set of 4 unexplained. + assert (evaluation.effective, evaluation.raw_quota, evaluation.raw_set_size) == (2.0, 2.0, 4) + assert evaluation.notices == () + + +def test_a_failed_memory_lookup_is_named_when_the_next_one_succeeds( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Names the `/proc` file the memory reading could not read, though the lookups after it in the call succeed.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + files={ + 'memory.max': '536870912\n', + 'memory.current': '150000000\n', + 'memory.stat': 'inactive_file 50000000\n', + 'cpu.max': '200000 100000\n', + 'cpu.stat': 'usage_usec 0\n', + }, + ) + refused = _cgroup._PROC_SELF_CGROUP + fail_the_first_lookup(monkeypatch) + + description = proclimits.describe() + + # `describe()` reads memory first, so that reading takes the one failure. The CPU reading after it locates + # its controllers and comes back as usual. + assert [(str(notice.code), notice.message) for notice in description.notices] == [ + ( + 'memory-metrics-unreadable', + ( + f'Where a memory limit would be read from cannot be located, because {refused} (EMFILE) cannot be ' + 'read. Nothing is reported, and the next reading tries again.' + ), + ), + ] + assert (description.memory_budget, description.raw_memory_limit) == (None, None) + assert description.cpu_limit == 2.0 + assert description.cpu_quota_source is not None + # The sources are located after the readings, so a lookup that succeeded since fills this one. + assert description.memory_source is not None + assert proclimits.get_memory_budget() == proclimits.MemoryBudget( + limit=536870912, used=100000000, available=436870912 + ) + + +@pytest.mark.parametrize( + ('reading', 'code'), + [ + pytest.param('read_memory', 'memory-metrics-unreadable', id='memory'), + pytest.param('read_cpu', 'cpu-metrics-unreadable', id='cpu'), + ], +) +def test_a_cache_cleared_after_a_reading_is_not_a_missing_mechanism( + fake_cgroup: Callable[..., Path], + monkeypatch: pytest.MonkeyPatch, + reading: str, + code: str, +) -> None: + """Names the failed lookup, rather than a missing mechanism, where the cache is cleared and `/proc` then fails.""" + fake_cgroup( + mountinfo=V2_MOUNTINFO, + self_cgroup=V2_SELF_CGROUP.format(path='/'), + # Controllers that limit nothing, so each reading goes on to ask whether a mechanism is here at all. + files={ + 'memory.current': '1\n', + 'memory.stat': 'inactive_file 0\n', + 'cpu.max': 'max 100000\n', + 'cpu.stat': 'usage_usec 1\n', + }, + ) + read = getattr(_backend, reading) + + def read_then_clear() -> _types.RawMemory | _types.RawCpu: + raw = read() + # Another thread forgets the discovery here, and the next `/proc` read fails. + _cgroup.clear_cache() + fail_the_first_lookup(monkeypatch) + return raw + + monkeypatch.setattr(_backend, reading, read_then_clear) + + assert [str(notice.code) for notice in proclimits.describe().notices] == [code] + + +@pytest.mark.usefixtures('_no_cgroup') +def test_describe_no_mechanism_messages() -> None: + """Says that nothing here carries a limit, which is a fact about the machine rather than a failed read.""" + messages = {str(notice.code): notice.message for notice in proclimits.describe().notices} + + assert messages == { + 'memory-metrics-unavailable': 'No mechanism here carries a memory limit, so nothing was read. A Linux ' + 'machine with no cgroup filesystem mounted looks like this, and so does a platform with no backend here.', + 'cpu-metrics-unavailable': 'No mechanism here carries a CPU limit, so nothing was read. A Linux machine ' + 'with no cgroup filesystem mounted looks like this, and so does a platform with no backend here.', + } + + @pytest.mark.parametrize( ('content', 'expected'), [ @@ -1235,8 +1465,8 @@ def test_no_rate_without_a_counter_for_the_limit( files={'cpu.max': '200000 100000\n', 'cpu.stat': 'usage_usec 0\n'}, ) quota = _cgroup.RawCpuQuota(cores=2.0, limit_level='/sys/fs/cgroup', usage_level=None) - monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda: quota) - monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda: None) + monkeypatch.setattr(_cgroup, 'read_cpu_quota', lambda _controllers: quota) + monkeypatch.setattr(_cgroup, 'read_cpu_set_size', lambda _controllers: None) # The limit itself still applies and is still reported. assert proclimits.get_cpu_limit() == 2.0 diff --git a/tests/unit/test_windows.py b/tests/unit/test_windows.py index 665f133..aaac7a8 100644 --- a/tests/unit/test_windows.py +++ b/tests/unit/test_windows.py @@ -136,21 +136,30 @@ def test_read_memory_without_a_job(monkeypatch: pytest.MonkeyPatch) -> None: """Reports no limit, and no failure, where nothing put this process in a job.""" monkeypatch.setattr(_windows, '_query', without_a_job()) - assert _windows.read_memory() == _windows._NO_MEMORY + raw = _windows.read_memory() + + assert raw == _windows._NO_MEMORY + # Equality with the constant does not pin the constant's own field. A job is not located through `/proc`. + assert raw.unlocated_file is None def test_read_memory_when_the_job_says_nothing(monkeypatch: pytest.MonkeyPatch) -> None: """Calls a job that will not answer unreadable, rather than unlimited.""" monkeypatch.setattr(_windows, '_query', answering()) - assert _windows.read_memory().unreadable_level == 'job' + raw = _windows.read_memory() + + assert (raw.unreadable_level, raw.unlocated_file) == ('job', None) def test_read_memory_without_a_memory_limit(monkeypatch: pytest.MonkeyPatch) -> None: """Reports nothing where the job answers and carries no memory limit.""" monkeypatch.setattr(_windows, '_query', answering(_JOB_EXTENDED_LIMIT=extended_limits())) - assert _windows.read_memory() == _windows._NO_MEMORY + raw = _windows.read_memory() + + assert raw == _windows._NO_MEMORY + assert raw.unlocated_file is None def test_read_memory_falls_back_to_the_distance(monkeypatch: pytest.MonkeyPatch) -> None: @@ -166,6 +175,7 @@ def test_read_memory_falls_back_to_the_distance(monkeypatch: pytest.MonkeyPatch) raw = _windows.read_memory() assert (raw.limit, raw.used, raw.available) == (512 * MIB, 12 * MIB, 500 * MIB) + assert raw.unlocated_file is None def test_read_memory_keeps_what_the_mechanism_answered(monkeypatch: pytest.MonkeyPatch) -> None: @@ -227,6 +237,7 @@ def test_read_memory_with_only_a_per_process_limit(monkeypatch: pytest.MonkeyPat assert raw.limit_level == 'job' # No call failed: a per-process limit is a shape the job pairs no usage figure with. assert raw.usage_unreadable_level is None + assert raw.unlocated_file is None def test_read_memory_prefers_a_tighter_per_process_limit(monkeypatch: pytest.MonkeyPatch) -> None: @@ -275,14 +286,16 @@ def test_read_cpu_without_a_job(monkeypatch: pytest.MonkeyPatch) -> None: raw = _windows.read_cpu() - assert (raw.quota, raw.cpu_set, raw.unreadable_level) == (None, None, None) + assert (raw.quota, raw.cpu_set, raw.unreadable_level, raw.unlocated_file) == (None, None, None, None) def test_read_cpu_when_the_job_says_nothing(monkeypatch: pytest.MonkeyPatch) -> None: """Calls a job that will not answer unreadable here too, so no looser answer is invented.""" monkeypatch.setattr(_windows, '_query', answering()) - assert _windows.read_cpu().unreadable_level == 'job' + raw = _windows.read_cpu() + + assert (raw.unreadable_level, raw.unlocated_file) == ('job', None) def test_read_cpu_takes_both_restrictions(monkeypatch: pytest.MonkeyPatch) -> None: @@ -304,6 +317,7 @@ def test_read_cpu_takes_both_restrictions(monkeypatch: pytest.MonkeyPatch) -> No assert raw.quota.cores == 1.5 assert raw.cpu_set is not None assert raw.cpu_set.cores == 3 + assert raw.unlocated_file is None def test_read_cpu_without_the_machine_cores(monkeypatch: pytest.MonkeyPatch) -> None: @@ -322,7 +336,7 @@ def test_read_cpu_without_the_machine_cores(monkeypatch: pytest.MonkeyPatch) -> raw = _windows.read_cpu() assert raw.unconvertible_level == 'job' - assert (raw.quota, raw.cpu_set, raw.unreadable_level) == (None, None, None) + assert (raw.quota, raw.cpu_set, raw.unreadable_level, raw.unlocated_file) == (None, None, None, None) def test_read_cpu_keeps_an_affinity_without_the_machine_cores(monkeypatch: pytest.MonkeyPatch) -> None: @@ -355,7 +369,7 @@ def test_read_cpu_when_only_the_rate_control_is_silent(monkeypatch: pytest.Monke raw = _windows.read_cpu() - assert (raw.quota, raw.cpu_set, raw.unreadable_level) == (None, None, 'job') + assert (raw.quota, raw.cpu_set, raw.unreadable_level, raw.unlocated_file) == (None, None, 'job', None) def test_machine_cpu_count_falls_back_on_one_group(monkeypatch: pytest.MonkeyPatch) -> None: @@ -396,7 +410,7 @@ def test_read_cpu_when_the_affinity_cannot_be_counted(monkeypatch: pytest.Monkey raw = _windows.read_cpu() - assert (raw.quota, raw.cpu_set, raw.unreadable_level) == (None, None, 'job') + assert (raw.quota, raw.cpu_set, raw.unreadable_level, raw.unlocated_file) == (None, None, 'job', None) def test_read_cpu_ignores_the_single_group_mask(monkeypatch: pytest.MonkeyPatch) -> None: @@ -458,6 +472,7 @@ def test_sources_without_a_job(monkeypatch: pytest.MonkeyPatch) -> None: assert sources.memory is not None assert sources.memory.levels == () + assert sources.unlocated_file is None def test_in_job_without_a_job(monkeypatch: pytest.MonkeyPatch) -> None: