From 16746745eba8128bb5ea8a75aff1a822453dc804 Mon Sep 17 00:00:00 2001 From: Prakriti Gupta Date: Sun, 1 Nov 2026 22:25:30 -0800 Subject: [PATCH] Libby keygrabber updates for logging --- docs/source/keygrabber.md | 4 ++ libby/cli/libby_cli.py | 4 +- libby/daemon.py | 41 +++++++------- libby/keygrabber/main.py | 18 +++++++ tests/test_cli_show.py | 75 ++++++++++++++++++++++++++ tests/test_daemon_logging.py | 102 +++++++++++++++++++++++++++++++++++ 6 files changed, 221 insertions(+), 23 deletions(-) create mode 100644 tests/test_cli_show.py create mode 100644 tests/test_daemon_logging.py diff --git a/docs/source/keygrabber.md b/docs/source/keygrabber.md index aec470c..4790ec5 100644 --- a/docs/source/keygrabber.md +++ b/docs/source/keygrabber.md @@ -24,6 +24,10 @@ It is an ordinary `LibbyDaemon`, so `SIGTERM` stops it cleanly and so does writing its `shutdown` keyword. On the way out it gives its retry queue a bounded chance to drain, so a graceful stop does not lose the last tick. +It logs INFO and above to stderr, which systemd puts in the journal. A +`logging` section in the config (`level`, `file`, `format`, `propagate`) +replaces that default rather than adding to it. + ## Config A keygrabber config is an ordinary daemon config (`peer_id`, `group_id`, diff --git a/libby/cli/libby_cli.py b/libby/cli/libby_cli.py index de2d792..0a127a2 100644 --- a/libby/cli/libby_cli.py +++ b/libby/cli/libby_cli.py @@ -334,8 +334,10 @@ def cmd_show(namespace: argparse.Namespace) -> int: # keep working against daemons still running a libby without it, # and a human reading a handful of keywords gains nothing from the # round trip saved + # The keyword is everything after ., so a dotted + # one like the keygrabber's adc.lastsample keeps its prefix rows: List[Tuple[str, Dict[str, Any]]] = [ - (name, _rpc_show_one(lib, peer, name.rsplit(".", 1)[-1], timeout)) + (name, _rpc_show_one(lib, peer, parse_keyword(name)[2], timeout)) for name in matches ] return _emit_many(rows, as_json=namespace.json) diff --git a/libby/daemon.py b/libby/daemon.py index 4ab9e2b..24c0792 100644 --- a/libby/daemon.py +++ b/libby/daemon.py @@ -19,8 +19,10 @@ EvtHandler = Callable[[Payload], None] DaemonT = TypeVar("DaemonT", bound="LibbyDaemon") +DEFAULT_LOG_FORMAT = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" -class _LastErrorHandler(logging.Handler): + +class _LastErrorHandler(logging.Filter): """Captures a daemon's most recent ERROR+ log record. A daemon that only logs a failure locally (self.logger.error(...)) leaves @@ -31,14 +33,17 @@ class _LastErrorHandler(logging.Handler): """ def __init__(self, daemon: "LibbyDaemon") -> None: - super().__init__(level=logging.ERROR) + super().__init__() self._daemon = daemon + self._formatter = logging.Formatter() - def emit(self, record: logging.LogRecord) -> None: - try: - self._daemon._last_error = self.format(record) - except Exception: - pass + def filter(self, record: logging.LogRecord) -> bool: + if record.levelno >= logging.ERROR: + try: + self._daemon._last_error = self._formatter.format(record) + except Exception: + pass + return True class LibbyDaemon: @@ -222,32 +227,24 @@ def _setup_logging(self) -> None: handler.setLevel(level) handler.setFormatter( - logging.Formatter( - str( - raw.get( - "format", - "%(asctime)s - %(name)s - " - "%(levelname)s - %(message)s", - ) - ) - ) + logging.Formatter(str(raw.get("format", DEFAULT_LOG_FORMAT))) ) setattr(handler, "_libby_daemon_handler", True) self.logger.addHandler(handler) def _ensure_last_error_handler(self) -> None: - """Attach the lasterror-tracking handler to self.logger (idempotent). + """Attach the lasterror-tracking filter to self.logger (idempotent). Called from both __init__ (covers daemons built directly, e.g. in tests or examples) and _setup_logging (which may point self.logger at a different, peer_id-named Logger once config is loaded). """ - if any(getattr(handler, "_libby_lasterror_handler", False) - for handler in self.logger.handlers): + if any(getattr(log_filter, "_libby_lasterror_handler", False) + for log_filter in self.logger.filters): return - handler = _LastErrorHandler(self) - setattr(handler, "_libby_lasterror_handler", True) - self.logger.addHandler(handler) + log_filter = _LastErrorHandler(self) + setattr(log_filter, "_libby_lasterror_handler", True) + self.logger.addFilter(log_filter) ### Optional hooks diff --git a/libby/keygrabber/main.py b/libby/keygrabber/main.py index 200e70e..da3faef 100644 --- a/libby/keygrabber/main.py +++ b/libby/keygrabber/main.py @@ -6,9 +6,12 @@ from __future__ import annotations import argparse +import logging import sys +from collections.abc import Mapping from typing import List, Optional +from ..daemon import DEFAULT_LOG_FORMAT from ..errors import LibbyError from .daemon import KeygrabberDaemon @@ -28,6 +31,8 @@ def main(argv: Optional[List[str]] = None) -> int: try: daemon = KeygrabberDaemon.from_config_file(namespace.config, namespace.daemon_id) + if not isinstance(daemon.get_config("logging"), Mapping): + log_to_stderr() daemon.serve() except KeyboardInterrupt: return 0 @@ -37,5 +42,18 @@ def main(argv: Optional[List[str]] = None) -> int: return 0 +def log_to_stderr() -> None: + """Send INFO and above to stderr, which systemd puts in the journal. + + Only for a config with no ``logging`` section. One with a section has + already given the daemon its own handler, and a second one on the root + logger would print every line twice. + """ + logging.basicConfig(level=logging.INFO, format=DEFAULT_LOG_FORMAT) + # pika logs each connection step at INFO, which the transport's + # once-a-second reconnect loop would turn into a flood during an outage + logging.getLogger("pika").setLevel(logging.WARNING) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/tests/test_cli_show.py b/tests/test_cli_show.py new file mode 100644 index 0000000..b7a4a21 --- /dev/null +++ b/tests/test_cli_show.py @@ -0,0 +1,75 @@ +"""Unit tests for ``libby show`` with a pattern. No transport needed. + +The keys it asks a daemon for are visible through a recording stand-in for +``Libby``, which matters because a daemon drops a request for a key it does not +serve without replying, so a wrong key surfaces only as a timeout. +""" +from __future__ import annotations + +import io +import json +import unittest +from contextlib import redirect_stdout +from typing import Any, Dict, List +from unittest import mock + +from libby.cli import libby_cli + + +class _RecordingLibby: + """Stands in for Libby, serving a fixed set of keywords and logging requests.""" + + def __init__(self, values: Dict[str, Any]) -> None: + self._values = values + self.requested: List[str] = [] + + def rpc(self, peer_id: str, key: str, payload: Dict[str, Any], + ttl_ms: int = 8000) -> Dict[str, Any]: + """Answer keys.list and shows; time out on any other key.""" + # Arguments are ignored; the signature exists to match Libby + # pylint: disable=unused-argument + self.requested.append(key) + if key == "keys.list": + return {"status": "delivered", + "resp": {"ok": True, "matches": list(self._values)}} + if key in self._values: + return {"status": "delivered", + "resp": {"ok": True, "value": self._values[key]}} + return {"status": "timeout"} + + def stop(self) -> None: + """Nothing to release.""" + + +class ShowPatternTests(unittest.TestCase): + """Which keys a wildcard show asks for.""" + + def _show(self, pattern: str, lib: _RecordingLibby) -> List[Dict[str, Any]]: + namespace = libby_cli.build_parser().parse_args(["show", pattern, "--json"]) + out = io.StringIO() + with mock.patch.object(libby_cli, "load_cli_config", return_value={}), \ + mock.patch.object(libby_cli, "_mk_libby", return_value=lib), \ + redirect_stdout(out): + libby_cli.cmd_show(namespace) + return json.loads(out.getvalue()) + + def test_keeps_the_prefix_of_a_dotted_keyword(self): + """Ask for adc.lastsample, not lastsample, which would time out.""" + lib = _RecordingLibby({"adc.lastsample": "2026-11-02T05:18:42+00:00", + "yj1.lastsample": "2026-11-02T05:18:40+00:00"}) + rows = self._show("hispec.keygrabber.%.lastsample", lib) + + self.assertEqual(lib.requested[1:], ["adc.lastsample", "yj1.lastsample"]) + self.assertTrue(all(row["ok"] for row in rows)) + self.assertEqual(rows[0]["qualified"], "hispec.keygrabber.adc.lastsample") + + def test_reads_a_plain_keyword(self): + lib = _RecordingLibby({"readerrors": 65}) + rows = self._show("hispec.keygrabber.%", lib) + + self.assertEqual(lib.requested[1:], ["readerrors"]) + self.assertEqual(rows[0]["value"], 65) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_daemon_logging.py b/tests/test_daemon_logging.py new file mode 100644 index 0000000..fe91a31 --- /dev/null +++ b/tests/test_daemon_logging.py @@ -0,0 +1,102 @@ +"""Unit tests for where a daemon's log records go. No transport needed. + +A daemon without a ``logging`` section must still be heard: Python prints +warnings and errors on stderr only while a logger has no handlers, so the +``lasterror`` capture must not be one. The keygrabber's entry point adds INFO +on top, unless the config's own section already decides. +""" +from __future__ import annotations + +import logging +import unittest +from typing import Any, Dict, Optional +from unittest import mock + +from libby.daemon import LibbyDaemon +from libby.keygrabber import main as keygrabber_main + +# lasterror is read back from the daemon's own attribute +# pylint: disable=protected-access + + +class LastErrorCaptureTests(unittest.TestCase): + """The lasterror keyword's source, and what it leaves alone.""" + + def setUp(self) -> None: + # Loggers are global by name, so each test gets one of its own + self.daemon = LibbyDaemon.from_config({"peer_id": f"logtest-{self.id()}"}) + + def tearDown(self) -> None: + self.daemon.logger.filters.clear() + + def test_adds_no_handler_without_a_logging_section(self): + """Leave Python's default stderr output for warnings and errors working.""" + self.assertEqual(self.daemon.logger.handlers, []) + + def test_captures_the_latest_error(self): + self.daemon.logger.error("first") + self.daemon.logger.error("collection %s failed", "adc") + self.assertEqual(self.daemon._last_error, "collection adc failed") + + def test_ignores_records_below_error(self): + self.daemon.logger.warning("only a warning") + self.assertIsNone(self.daemon._last_error) + + def test_keeps_the_traceback_of_a_logged_exception(self): + try: + raise RuntimeError("hardware unreachable") + except RuntimeError: + self.daemon.logger.exception("daemon initialization failed") + self.assertIn("daemon initialization failed", self.daemon._last_error) + self.assertIn("RuntimeError: hardware unreachable", self.daemon._last_error) + + def test_attaches_once(self): + self.daemon._setup_logging() + self.assertEqual(len(self.daemon.logger.filters), 1) + + +class _StubDaemon: + """Stands in for KeygrabberDaemon, holding a config and returning at once.""" + + def __init__(self, config: Dict[str, Any]) -> None: + self._config = config + + def get_config(self, key: str, default: Any = None) -> Any: + """Return a top-level config value.""" + return self._config.get(key, default) + + def serve(self) -> None: + """Return immediately, as if stopped.""" + + +class KeygrabberEntryPointTests(unittest.TestCase): + """Whether ``keygrabber`` sends INFO to stderr itself.""" + + def _run(self, logging_section: Optional[Dict[str, Any]]) -> mock.Mock: + config = {} if logging_section is None else {"logging": logging_section} + with mock.patch.object(keygrabber_main.KeygrabberDaemon, "from_config_file", + return_value=_StubDaemon(config)), \ + mock.patch.object(keygrabber_main, "log_to_stderr") as log_to_stderr: + self.assertEqual(keygrabber_main.main(["-c", "keygrabber.yaml"]), 0) + return log_to_stderr + + def test_logs_to_stderr_without_a_logging_section(self): + self._run(None).assert_called_once_with() + + def test_leaves_a_configured_logging_section_alone(self): + """Do not add a root handler that would print every line twice.""" + self._run({"level": "DEBUG"}).assert_not_called() + + def test_quiets_pika_connection_chatter(self): + pika = logging.getLogger("pika") + level = pika.level + with mock.patch.object(keygrabber_main.logging, "basicConfig"): + keygrabber_main.log_to_stderr() + try: + self.assertEqual(pika.level, logging.WARNING) + finally: + pika.setLevel(level) + + +if __name__ == "__main__": + unittest.main()