From cda42ee10c040353d550ae0209d3c493718cd879 Mon Sep 17 00:00:00 2001 From: Don Neill Date: Tue, 29 Sep 2026 16:44:16 -0700 Subject: [PATCH 1/9] Initial Revision of keyheader: just dir and init file --- libby/keyheader/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 libby/keyheader/__init__.py diff --git a/libby/keyheader/__init__.py b/libby/keyheader/__init__.py new file mode 100644 index 0000000..e69de29 From 6989ea9a61f873b79babdd1049aab5ccdcbfe917 Mon Sep 17 00:00:00 2001 From: Don Neill Date: Wed, 30 Sep 2026 10:50:30 -0700 Subject: [PATCH 2/9] Added InfluxSource class and associated helper functions and tests. --- libby/keygrabber/influx.py | 108 +++++++++++- tests/test_influx.py | 328 +++++++++++++++++++++++++++++++++++++ tests/test_influx_sink.py | 167 ------------------- 3 files changed, 433 insertions(+), 170 deletions(-) create mode 100644 tests/test_influx.py delete mode 100644 tests/test_influx_sink.py diff --git a/libby/keygrabber/influx.py b/libby/keygrabber/influx.py index ff366b1..2834766 100644 --- a/libby/keygrabber/influx.py +++ b/libby/keygrabber/influx.py @@ -1,4 +1,4 @@ -"""InfluxDB 2.x sink. +"""InfluxDB 2.x sink, and a source that reads its samples back. Schema is one measurement per keyword name, tagged by ``group``, ``peer`` and ``units``, with a single ``value`` field. A keyword name carries one type across @@ -10,12 +10,14 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Optional, Sequence +from datetime import datetime, timezone +from typing import List, Optional, Sequence from influxdb_client import InfluxDBClient, Point, WritePrecision +from influxdb_client.client.flux_table import FluxRecord from influxdb_client.client.write_api import SYNCHRONOUS -from .sink import Sample, SinkWriteError, Value +from .sink import Sample, SinkError, SinkWriteError, Value # Influx drops an empty tag value, which would split one keyword into two # series depending on whether it declared units. A literal keeps every point @@ -121,3 +123,103 @@ def close(self) -> None: self._write_api = None if client is not None: client.close() + + +def flux_string(text: str) -> str: + """Quote text as a Flux string literal. + + Parameterized queries are Cloud-only, so values are escaped instead. ``$`` + is escaped too, since Flux would otherwise read ``${`` as interpolation. + """ + escaped = (text.replace("\\", "\\\\").replace('"', '\\"') + .replace("$", "\\$")) + return f'"{escaped}"' + + +def flux_time(moment: datetime) -> str: + """Format a datetime as a Flux RFC3339 time literal in UTC.""" + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def to_sample(record: FluxRecord) -> Sample: + """Map a queried record back to a sample, undoing :func:`to_point`.""" + units = record.values.get("units") + return Sample(keyword=record.get_measurement(), + group=record.values.get("group", ""), + peer=record.values.get("peer", ""), + value=record.get_value(), + units=None if units in (None, NO_UNITS) else units, + timestamp=record.get_time()) + + +class InfluxSource: + """Reads samples back from an InfluxDB 2.x bucket.""" + + def __init__(self, config: InfluxConfig) -> None: + self._config = config + self._client: Optional[InfluxDBClient] = None + self._query_api = None + + def connect(self) -> None: + """Open the client, replacing any existing one.""" + self.close() + self._client = InfluxDBClient(url=self._config.url, + token=self._config.token, + org=self._config.org, + timeout=self._config.timeout_ms) + self._query_api = self._client.query_api() + + def is_connected(self) -> bool: + """Return whether the server answers a ping.""" + if self._client is None: + return False + try: + return bool(self._client.ping()) + except Exception: # pylint: disable=broad-exception-caught + return False + + def read( + self, + keyword: str, + start: datetime, + stop: Optional[datetime] = None, + peer: Optional[str] = None, + ) -> List[Sample]: + """Return one keyword's samples in ``[start, stop)``, oldest first.""" + if self._query_api is None: + raise SinkError("influx source is not connected") + query = self._build_query(keyword, start, stop, peer) + try: + tables = self._query_api.query(query, org=self._config.org) + # The client surfaces API, HTTP and socket errors with no common base + except Exception as exc: # pylint: disable=broad-exception-caught + raise SinkError(f"influx query failed: {exc}") from exc + samples = [to_sample(record) for table in tables + for record in table.records] + samples.sort(key=lambda sample: sample.timestamp) + return samples + + def close(self) -> None: + """Release the client; safe to call when never connected.""" + client, self._client = self._client, None + self._query_api = None + if client is not None: + client.close() + + def _build_query( + self, + keyword: str, + start: datetime, + stop: Optional[datetime], + peer: Optional[str], + ) -> str: + time_range = f"start: {flux_time(start)}" + if stop is not None: + time_range += f", stop: {flux_time(stop)}" + predicate = (f"r._measurement == {flux_string(keyword)}" + ' and r._field == "value"') + if peer is not None: + predicate += f" and r.peer == {flux_string(peer)}" + return (f"from(bucket: {flux_string(self._config.bucket)})\n" + f" |> range({time_range})\n" + f" |> filter(fn: (r) => {predicate})") diff --git a/tests/test_influx.py b/tests/test_influx.py new file mode 100644 index 0000000..e2b181d --- /dev/null +++ b/tests/test_influx.py @@ -0,0 +1,328 @@ +"""Unit tests for the InfluxDB sink and source: sample mapping, queries, errors. + +Needs the optional dependency (``pip install libby[influxdb]``); tox installs +it, so these run in CI rather than skipping. +""" +from __future__ import annotations + +import importlib.util +import unittest +from datetime import datetime, timedelta, timezone +from typing import Any, List, Optional + +from libby.keygrabber import Sample, SinkError, SinkWriteError + +HAS_INFLUX = importlib.util.find_spec("influxdb_client") is not None + +# Every class below is skipUnless-guarded on HAS_INFLUX, which pylint +# cannot see through +# pylint: disable=possibly-used-before-assignment + +if HAS_INFLUX: + from influxdb_client.client.flux_table import FluxRecord + + # Grouped under the guard with the client import, not with libby's above + # pylint: disable-next=ungrouped-imports + from libby.keygrabber.influx import (NO_UNITS, InfluxConfig, InfluxSink, + InfluxSource, field_value, + flux_string, flux_time, to_point, + to_sample) + + +def _sample( + keyword: str = "positionvalue", + value: Any = 7.5, + units: Optional[str] = "mm", +) -> Sample: + return Sample(keyword=keyword, group="hsfei", peer="adc", value=value, + units=units, + timestamp=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) + + +class _FakeWriteApi: # pylint: disable=too-few-public-methods + """Records write calls, or raises to exercise the error contract.""" + + def __init__(self, error: Optional[Exception] = None) -> None: + self.records: List[Any] = [] + self._error = error + + def write(self, bucket: str, org: str, record: Any) -> None: + """Record the batch, or raise the configured error.""" + if self._error is not None: + raise self._error + self.records.append((bucket, org, record)) + + +def _record( + keyword: str = "positionvalue", + value: Any = 7.5, + units: Optional[str] = "mm", + peer: str = "adc", + minute: int = 0, +) -> "FluxRecord": + return FluxRecord(0, { + "_measurement": keyword, "_field": "value", "_value": value, + "_time": datetime(2026, 1, 1, 12, minute, tzinfo=timezone.utc), + "group": "hsfei", "peer": peer, "units": units, + }) + + +class _FakeTable: # pylint: disable=too-few-public-methods + """Stands in for a FluxTable, which the source only reads records from.""" + + def __init__(self, records: List[Any]) -> None: + self.records = records + + +class _FakeQueryApi: # pylint: disable=too-few-public-methods + """Records query calls and returns canned tables, or raises.""" + + def __init__(self, tables: Optional[List[_FakeTable]] = None, + error: Optional[Exception] = None) -> None: + self.queries: List[Any] = [] + self._tables = tables or [] + self._error = error + + def query(self, query: str, org: str) -> List[_FakeTable]: + """Record the query, then return the tables or raise the error.""" + self.queries.append((query, org)) + if self._error is not None: + raise self._error + return self._tables + + +@unittest.skipUnless(HAS_INFLUX, "influxdb-client not installed") +class FieldValueTests(unittest.TestCase): + """Coercion of keyword values to Influx field types.""" + + def test_int_becomes_float(self): + """Avoid an int/float type conflict between peers for one keyword.""" + coerced = field_value(12) + self.assertIsInstance(coerced, float) + self.assertEqual(coerced, 12.0) + + def test_bool_stays_bool(self): + """Keep bools as bools, despite bool being a subclass of int.""" + self.assertIs(field_value(True), True) + self.assertIsInstance(field_value(False), bool) + + def test_float_stays_float(self): + """Pass a float through unchanged.""" + self.assertEqual(field_value(7.5), 7.5) + + def test_string_stays_string(self): + """Store a string keyword as a string field.""" + self.assertEqual(field_value("Ready"), "Ready") + + +@unittest.skipUnless(HAS_INFLUX, "influxdb-client not installed") +class ToPointTests(unittest.TestCase): + """Mapping a sample onto the measurement-per-keyword schema.""" + + def test_measurement_is_the_keyword_name(self): + """Use the keyword name as the measurement, with group/peer as tags.""" + line = to_point(_sample()).to_line_protocol() + self.assertTrue(line.startswith("positionvalue,")) + self.assertIn("group=hsfei", line) + self.assertIn("peer=adc", line) + self.assertIn("units=mm", line) + self.assertIn("value=7.5", line) + + def test_null_value_is_skipped(self): + """Skip a null rather than raising: lasterror is null most of the time.""" + self.assertIsNone(to_point(_sample(keyword="lasterror", value=None))) + + def test_missing_units_get_a_placeholder(self): + """Keep a unitless keyword on one series instead of dropping the tag.""" + line = to_point(_sample(units=None)).to_line_protocol() + self.assertIn(f"units={NO_UNITS}", line) + + def test_timestamp_is_carried_at_nanosecond_precision(self): + """Stamp the point with the sample's own read time.""" + line = to_point(_sample()).to_line_protocol() + expected_ns = int(_sample().timestamp.timestamp() * 1_000_000_000) + self.assertTrue(line.endswith(str(expected_ns))) + + def test_string_value_is_quoted_by_the_client(self): + """Let the client escape a string field rather than hand-rolling it.""" + line = to_point(_sample(keyword="lasterror", + value='failed: "x", y', units=None)).to_line_protocol() + self.assertIn('value="failed: \\"x\\", y"', line) + + +@unittest.skipUnless(HAS_INFLUX, "influxdb-client not installed") +class InfluxSinkWriteTests(unittest.TestCase): + """The sink's write contract, without a server.""" + + # The write api is injected in place of connect() so these need no server + # pylint: disable=protected-access + + def setUp(self) -> None: + self.sink = InfluxSink(InfluxConfig( + url="http://localhost:8086", org="hispec", + bucket="telemetry", token="unused")) + + def test_writes_one_request_for_the_whole_batch(self): + """Send a batch as a single request and report the count written.""" + api = _FakeWriteApi() + self.sink._write_api = api + written = self.sink.write([_sample(), _sample(keyword="isconnected", + value=True, units=None)]) + self.assertEqual(written, 2) + self.assertEqual(len(api.records), 1) + self.assertEqual(len(api.records[0][2]), 2) + + def test_nulls_are_not_counted_as_written(self): + """Report only the samples that reached the backend.""" + api = _FakeWriteApi() + self.sink._write_api = api + written = self.sink.write([_sample(), + _sample(keyword="lasterror", value=None)]) + self.assertEqual(written, 1) + + def test_all_null_batch_makes_no_request(self): + """Skip the request entirely when nothing is storable.""" + api = _FakeWriteApi() + self.sink._write_api = api + self.assertEqual(self.sink.write([_sample(value=None)]), 0) + self.assertEqual(api.records, []) + + def test_backend_failure_becomes_a_sink_write_error(self): + """Translate the client's own exceptions into one retryable error.""" + self.sink._write_api = _FakeWriteApi(error=OSError("connection refused")) + with self.assertRaises(SinkWriteError): + self.sink.write([_sample()]) + + def test_writing_before_connect_is_a_sink_write_error(self): + """Fail a write on an unconnected sink the same retryable way.""" + with self.assertRaises(SinkWriteError): + self.sink.write([_sample()]) + + def test_is_connected_is_false_before_connect(self): + """Report not connected rather than raising.""" + self.assertFalse(self.sink.is_connected()) + + def test_close_without_connect_is_safe(self): + """Allow teardown of a sink that never opened.""" + self.sink.close() + + +@unittest.skipUnless(HAS_INFLUX, "influxdb-client not installed") +class FluxLiteralTests(unittest.TestCase): + """Quoting values into Flux without parameterized queries.""" + + def test_plain_string_is_quoted(self): + """Wrap an ordinary name in double quotes.""" + self.assertEqual(flux_string("adc"), '"adc"') + + def test_quote_and_backslash_are_escaped(self): + """Keep a quote or backslash from ending the literal early.""" + self.assertEqual(flux_string('a"b\\c'), '"a\\"b\\\\c"') + + def test_dollar_is_escaped(self): + """Stop Flux reading ${ as string interpolation.""" + self.assertEqual(flux_string("${x}"), '"\\${x}"') + + def test_time_is_converted_to_utc(self): + """Format an offset-aware time as a UTC RFC3339 literal.""" + pacific = timezone(timedelta(hours=-7)) + moment = datetime(2026, 1, 1, 5, 0, tzinfo=pacific) + self.assertEqual(flux_time(moment), "2026-01-01T12:00:00.000000Z") + + +@unittest.skipUnless(HAS_INFLUX, "influxdb-client not installed") +class ToSampleTests(unittest.TestCase): + """Mapping a queried record back onto a sample.""" + + def test_record_round_trips_to_a_sample(self): + """Recover the sample that to_point stored.""" + self.assertEqual(to_sample(_record()), _sample()) + + def test_units_placeholder_becomes_none(self): + """Undo the placeholder that keeps unitless keywords on one series.""" + self.assertIsNone(to_sample(_record(units=NO_UNITS)).units) + + +@unittest.skipUnless(HAS_INFLUX, "influxdb-client not installed") +class InfluxSourceReadTests(unittest.TestCase): + """The source's read contract, without a server.""" + + # The query api is injected in place of connect() so these need no server + # pylint: disable=protected-access + + START = datetime(2026, 1, 1, 11, 0, tzinfo=timezone.utc) + STOP = datetime(2026, 1, 1, 13, 0, tzinfo=timezone.utc) + + def setUp(self) -> None: + self.source = InfluxSource(InfluxConfig( + url="http://localhost:8086", org="hispec", + bucket="telemetry", token="unused")) + + def _query(self, **kwargs: Any) -> str: + api = _FakeQueryApi() + self.source._query_api = api + self.source.read("positionvalue", self.START, **kwargs) + self.assertEqual(api.queries[0][1], "hispec") + return api.queries[0][0] + + def test_query_selects_bucket_measurement_and_field(self): + """Query the configured bucket for the keyword's value field.""" + query = self._query() + self.assertIn('from(bucket: "telemetry")', query) + self.assertIn('r._measurement == "positionvalue"', query) + self.assertIn('r._field == "value"', query) + self.assertNotIn("r.peer", query) + + def test_open_ended_range_has_no_stop(self): + """Leave stop out so the range runs up to now.""" + query = self._query() + self.assertIn("range(start: 2026-01-01T11:00:00.000000Z)", query) + + def test_stop_bounds_the_range(self): + """Pass an explicit stop through to the range.""" + query = self._query(stop=self.STOP) + self.assertIn("stop: 2026-01-01T13:00:00.000000Z", query) + + def test_peer_filter_is_escaped(self): + """Filter on peer, quoting it so it cannot alter the query.""" + query = self._query(peer='adc" or true') + self.assertIn('r.peer == "adc\\" or true"', query) + + def test_records_across_tables_come_back_oldest_first(self): + """Merge every table's records, sorted by time.""" + self.source._query_api = _FakeQueryApi(tables=[ + _FakeTable([_record(peer="adc", minute=30)]), + _FakeTable([_record(peer="fei", minute=10), + _record(peer="fei", minute=20)]), + ]) + samples = self.source.read("positionvalue", self.START) + self.assertEqual([(s.peer, s.timestamp.minute) for s in samples], + [("fei", 10), ("fei", 20), ("adc", 30)]) + + def test_no_data_is_an_empty_list(self): + """Return nothing rather than raising when the range is empty.""" + self.source._query_api = _FakeQueryApi() + self.assertEqual(self.source.read("positionvalue", self.START), []) + + def test_backend_failure_becomes_a_sink_error(self): + """Translate the client's own exceptions into one error type.""" + self.source._query_api = _FakeQueryApi(error=OSError("refused")) + with self.assertRaises(SinkError): + self.source.read("positionvalue", self.START) + + def test_reading_before_connect_is_a_sink_error(self): + """Fail a read on an unconnected source rather than on None.""" + with self.assertRaises(SinkError): + self.source.read("positionvalue", self.START) + + def test_is_connected_is_false_before_connect(self): + """Report not connected rather than raising.""" + self.assertFalse(self.source.is_connected()) + + def test_close_without_connect_is_safe(self): + """Allow teardown of a source that never opened.""" + self.source.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_influx_sink.py b/tests/test_influx_sink.py deleted file mode 100644 index 0fb7401..0000000 --- a/tests/test_influx_sink.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Unit tests for the InfluxDB sink's sample mapping and error contract. - -Needs the optional dependency (``pip install libby[influxdb]``); tox installs -it, so these run in CI rather than skipping. -""" -from __future__ import annotations - -import importlib.util -import unittest -from datetime import datetime, timezone -from typing import Any, List, Optional - -from libby.keygrabber import Sample, SinkWriteError - -HAS_INFLUX = importlib.util.find_spec("influxdb_client") is not None - -# Every class below is skipUnless-guarded on HAS_INFLUX, which pylint -# cannot see through -# pylint: disable=possibly-used-before-assignment - -if HAS_INFLUX: - from libby.keygrabber.influx import (NO_UNITS, InfluxConfig, InfluxSink, - field_value, to_point) - - -def _sample( - keyword: str = "positionvalue", - value: Any = 7.5, - units: Optional[str] = "mm", -) -> Sample: - return Sample(keyword=keyword, group="hsfei", peer="adc", value=value, - units=units, - timestamp=datetime(2026, 1, 1, 12, 0, tzinfo=timezone.utc)) - - -class _FakeWriteApi: # pylint: disable=too-few-public-methods - """Records write calls, or raises to exercise the error contract.""" - - def __init__(self, error: Optional[Exception] = None) -> None: - self.records: List[Any] = [] - self._error = error - - def write(self, bucket: str, org: str, record: Any) -> None: - """Record the batch, or raise the configured error.""" - if self._error is not None: - raise self._error - self.records.append((bucket, org, record)) - - -@unittest.skipUnless(HAS_INFLUX, "influxdb-client not installed") -class FieldValueTests(unittest.TestCase): - """Coercion of keyword values to Influx field types.""" - - def test_int_becomes_float(self): - """Avoid an int/float type conflict between peers for one keyword.""" - coerced = field_value(12) - self.assertIsInstance(coerced, float) - self.assertEqual(coerced, 12.0) - - def test_bool_stays_bool(self): - """Keep bools as bools, despite bool being a subclass of int.""" - self.assertIs(field_value(True), True) - self.assertIsInstance(field_value(False), bool) - - def test_float_stays_float(self): - """Pass a float through unchanged.""" - self.assertEqual(field_value(7.5), 7.5) - - def test_string_stays_string(self): - """Store a string keyword as a string field.""" - self.assertEqual(field_value("Ready"), "Ready") - - -@unittest.skipUnless(HAS_INFLUX, "influxdb-client not installed") -class ToPointTests(unittest.TestCase): - """Mapping a sample onto the measurement-per-keyword schema.""" - - def test_measurement_is_the_keyword_name(self): - """Use the keyword name as the measurement, with group/peer as tags.""" - line = to_point(_sample()).to_line_protocol() - self.assertTrue(line.startswith("positionvalue,")) - self.assertIn("group=hsfei", line) - self.assertIn("peer=adc", line) - self.assertIn("units=mm", line) - self.assertIn("value=7.5", line) - - def test_null_value_is_skipped(self): - """Skip a null rather than raising: lasterror is null most of the time.""" - self.assertIsNone(to_point(_sample(keyword="lasterror", value=None))) - - def test_missing_units_get_a_placeholder(self): - """Keep a unitless keyword on one series instead of dropping the tag.""" - line = to_point(_sample(units=None)).to_line_protocol() - self.assertIn(f"units={NO_UNITS}", line) - - def test_timestamp_is_carried_at_nanosecond_precision(self): - """Stamp the point with the sample's own read time.""" - line = to_point(_sample()).to_line_protocol() - expected_ns = int(_sample().timestamp.timestamp() * 1_000_000_000) - self.assertTrue(line.endswith(str(expected_ns))) - - def test_string_value_is_quoted_by_the_client(self): - """Let the client escape a string field rather than hand-rolling it.""" - line = to_point(_sample(keyword="lasterror", - value='failed: "x", y', units=None)).to_line_protocol() - self.assertIn('value="failed: \\"x\\", y"', line) - - -@unittest.skipUnless(HAS_INFLUX, "influxdb-client not installed") -class InfluxSinkWriteTests(unittest.TestCase): - """The sink's write contract, without a server.""" - - # The write api is injected in place of connect() so these need no server - # pylint: disable=protected-access - - def setUp(self) -> None: - self.sink = InfluxSink(InfluxConfig( - url="http://localhost:8086", org="hispec", - bucket="telemetry", token="unused")) - - def test_writes_one_request_for_the_whole_batch(self): - """Send a batch as a single request and report the count written.""" - api = _FakeWriteApi() - self.sink._write_api = api - written = self.sink.write([_sample(), _sample(keyword="isconnected", - value=True, units=None)]) - self.assertEqual(written, 2) - self.assertEqual(len(api.records), 1) - self.assertEqual(len(api.records[0][2]), 2) - - def test_nulls_are_not_counted_as_written(self): - """Report only the samples that reached the backend.""" - api = _FakeWriteApi() - self.sink._write_api = api - written = self.sink.write([_sample(), - _sample(keyword="lasterror", value=None)]) - self.assertEqual(written, 1) - - def test_all_null_batch_makes_no_request(self): - """Skip the request entirely when nothing is storable.""" - api = _FakeWriteApi() - self.sink._write_api = api - self.assertEqual(self.sink.write([_sample(value=None)]), 0) - self.assertEqual(api.records, []) - - def test_backend_failure_becomes_a_sink_write_error(self): - """Translate the client's own exceptions into one retryable error.""" - self.sink._write_api = _FakeWriteApi(error=OSError("connection refused")) - with self.assertRaises(SinkWriteError): - self.sink.write([_sample()]) - - def test_writing_before_connect_is_a_sink_write_error(self): - """Fail a write on an unconnected sink the same retryable way.""" - with self.assertRaises(SinkWriteError): - self.sink.write([_sample()]) - - def test_is_connected_is_false_before_connect(self): - """Report not connected rather than raising.""" - self.assertFalse(self.sink.is_connected()) - - def test_close_without_connect_is_safe(self): - """Allow teardown of a sink that never opened.""" - self.sink.close() - - -if __name__ == "__main__": - unittest.main() From e7126f3a97a145100586fe3932c85c1ee8811dca Mon Sep 17 00:00:00 2001 From: Don Neill Date: Wed, 30 Sep 2026 11:34:22 -0700 Subject: [PATCH 3/9] Create source.py in keyheader dir and update keygrabber/influx.py to use source errors in source class. --- libby/keygrabber/influx.py | 10 ++++---- libby/keyheader/source.py | 49 ++++++++++++++++++++++++++++++++++++++ tests/test_influx.py | 13 +++++----- 3 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 libby/keyheader/source.py diff --git a/libby/keygrabber/influx.py b/libby/keygrabber/influx.py index 2834766..057b9b7 100644 --- a/libby/keygrabber/influx.py +++ b/libby/keygrabber/influx.py @@ -17,7 +17,8 @@ from influxdb_client.client.flux_table import FluxRecord from influxdb_client.client.write_api import SYNCHRONOUS -from .sink import Sample, SinkError, SinkWriteError, Value +from ..keyheader.source import SourceReadError +from .sink import Sample, SinkWriteError, Value # Influx drops an empty tag value, which would split one keyword into two # series depending on whether it declared units. A literal keeps every point @@ -187,13 +188,14 @@ def read( ) -> List[Sample]: """Return one keyword's samples in ``[start, stop)``, oldest first.""" if self._query_api is None: - raise SinkError("influx source is not connected") + raise SourceReadError("influx source is not connected") query = self._build_query(keyword, start, stop, peer) try: tables = self._query_api.query(query, org=self._config.org) - # The client surfaces API, HTTP and socket errors with no common base + # The client surfaces API, HTTP and socket errors with no common base, + # and the caller's contract is a single retryable error except Exception as exc: # pylint: disable=broad-exception-caught - raise SinkError(f"influx query failed: {exc}") from exc + raise SourceReadError(f"influx query failed: {exc}") from exc samples = [to_sample(record) for table in tables for record in table.records] samples.sort(key=lambda sample: sample.timestamp) diff --git a/libby/keyheader/source.py b/libby/keyheader/source.py new file mode 100644 index 0000000..bf6b51d --- /dev/null +++ b/libby/keyheader/source.py @@ -0,0 +1,49 @@ +"""The source contract keyheader reads stored keyword samples from. + +A source hands back the same :class:`~libby.keygrabber.sink.Sample` the +keygrabber wrote, so turning measurements, fields, tags or columns back into +samples is the source's job, and a second backend is a new source rather than +a change to keyheader. +""" +from __future__ import annotations + +from datetime import datetime +from typing import List, Optional, Protocol + +from ..errors import LibbyError +from ..keygrabber.sink import Sample + + +class SourceError(LibbyError): + """A source could not be reached, configured, or read from.""" + + +class SourceReadError(SourceError): + """A query for samples failed and may be worth retrying.""" + + +class Source(Protocol): + """Origin of stored samples. + + Implementations own their own schema. ``read`` returns an empty list when + nothing was stored in the range, and raises :class:`SourceReadError` when + the query itself failed. + """ + + def connect(self) -> None: + """Open the connection, or reopen it after a failure.""" + + def is_connected(self) -> bool: + """Return whether the backend is currently reachable.""" + + def read( + self, + keyword: str, + start: datetime, + stop: Optional[datetime] = None, + peer: Optional[str] = None, + ) -> List[Sample]: + """Return one keyword's samples in ``[start, stop)``, oldest first.""" + + def close(self) -> None: + """Release the connection.""" diff --git a/tests/test_influx.py b/tests/test_influx.py index e2b181d..6cba847 100644 --- a/tests/test_influx.py +++ b/tests/test_influx.py @@ -10,7 +10,8 @@ from datetime import datetime, timedelta, timezone from typing import Any, List, Optional -from libby.keygrabber import Sample, SinkError, SinkWriteError +from libby.keygrabber import Sample, SinkWriteError +from libby.keyheader.source import SourceReadError HAS_INFLUX = importlib.util.find_spec("influxdb_client") is not None @@ -304,15 +305,15 @@ def test_no_data_is_an_empty_list(self): self.source._query_api = _FakeQueryApi() self.assertEqual(self.source.read("positionvalue", self.START), []) - def test_backend_failure_becomes_a_sink_error(self): - """Translate the client's own exceptions into one error type.""" + def test_backend_failure_becomes_a_source_read_error(self): + """Translate the client's own exceptions into one retryable error.""" self.source._query_api = _FakeQueryApi(error=OSError("refused")) - with self.assertRaises(SinkError): + with self.assertRaises(SourceReadError): self.source.read("positionvalue", self.START) - def test_reading_before_connect_is_a_sink_error(self): + def test_reading_before_connect_is_a_source_read_error(self): """Fail a read on an unconnected source rather than on None.""" - with self.assertRaises(SinkError): + with self.assertRaises(SourceReadError): self.source.read("positionvalue", self.START) def test_is_connected_is_false_before_connect(self): From b8deb34b2e17976bd11a64aef48292eb593c1b42 Mon Sep 17 00:00:00 2001 From: Don Neill Date: Wed, 30 Sep 2026 13:00:32 -0700 Subject: [PATCH 4/9] Add group to query and associated methods --- libby/keygrabber/influx.py | 6 +++++- libby/keyheader/source.py | 7 ++++++- tests/test_influx.py | 12 ++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/libby/keygrabber/influx.py b/libby/keygrabber/influx.py index 057b9b7..c2e5166 100644 --- a/libby/keygrabber/influx.py +++ b/libby/keygrabber/influx.py @@ -184,12 +184,13 @@ def read( keyword: str, start: datetime, stop: Optional[datetime] = None, + group: Optional[str] = None, peer: Optional[str] = None, ) -> List[Sample]: """Return one keyword's samples in ``[start, stop)``, oldest first.""" if self._query_api is None: raise SourceReadError("influx source is not connected") - query = self._build_query(keyword, start, stop, peer) + query = self._build_query(keyword, start, stop, group, peer) try: tables = self._query_api.query(query, org=self._config.org) # The client surfaces API, HTTP and socket errors with no common base, @@ -213,6 +214,7 @@ def _build_query( keyword: str, start: datetime, stop: Optional[datetime], + group: Optional[str], peer: Optional[str], ) -> str: time_range = f"start: {flux_time(start)}" @@ -220,6 +222,8 @@ def _build_query( time_range += f", stop: {flux_time(stop)}" predicate = (f"r._measurement == {flux_string(keyword)}" ' and r._field == "value"') + if group is not None: + predicate += f" and r.group == {flux_string(group)}" if peer is not None: predicate += f" and r.peer == {flux_string(peer)}" return (f"from(bucket: {flux_string(self._config.bucket)})\n" diff --git a/libby/keyheader/source.py b/libby/keyheader/source.py index bf6b51d..088d1ea 100644 --- a/libby/keyheader/source.py +++ b/libby/keyheader/source.py @@ -41,9 +41,14 @@ def read( keyword: str, start: datetime, stop: Optional[datetime] = None, + group: Optional[str] = None, peer: Optional[str] = None, ) -> List[Sample]: - """Return one keyword's samples in ``[start, stop)``, oldest first.""" + """Return one keyword's samples in ``[start, stop)``, oldest first. + + ``group`` and ``peer`` each narrow the result when given. A peer name + alone can match daemons in several groups, so pass both to select one. + """ def close(self) -> None: """Release the connection.""" diff --git a/tests/test_influx.py b/tests/test_influx.py index 6cba847..237d8b1 100644 --- a/tests/test_influx.py +++ b/tests/test_influx.py @@ -272,6 +272,7 @@ def test_query_selects_bucket_measurement_and_field(self): self.assertIn('from(bucket: "telemetry")', query) self.assertIn('r._measurement == "positionvalue"', query) self.assertIn('r._field == "value"', query) + self.assertNotIn("r.group", query) self.assertNotIn("r.peer", query) def test_open_ended_range_has_no_stop(self): @@ -289,6 +290,17 @@ def test_peer_filter_is_escaped(self): query = self._query(peer='adc" or true') self.assertIn('r.peer == "adc\\" or true"', query) + def test_group_filter_is_escaped(self): + """Filter on group, quoting it so it cannot alter the query.""" + query = self._query(group='hsfei" or true') + self.assertIn('r.group == "hsfei\\" or true"', query) + self.assertNotIn("r.peer", query) + + def test_group_and_peer_filter_together(self): + """Narrow to one daemon when its name is reused across groups.""" + query = self._query(group="hsfei", peer="adc") + self.assertIn('r.group == "hsfei" and r.peer == "adc"', query) + def test_records_across_tables_come_back_oldest_first(self): """Merge every table's records, sorted by time.""" self.source._query_api = _FakeQueryApi(tables=[ From 8919a89693a1b796876e231d3ec9c01fe8f5ef23 Mon Sep 17 00:00:00 2001 From: Don Neill Date: Wed, 11 Nov 2026 15:48:33 -0800 Subject: [PATCH 5/9] Add the keyheader daemon: FITS header segments from stored keywords A requesting daemon asks for a named segment at a moment and gets back {keyword, value, comment} entries ready for astropy. Values come from the keygrabber's database, chosen per card by interpolate, nearest, prior or subsequent within a window. Config load rejects malformed or FITS-reserved keywords, and any FITS keyword repeated within or across segments. Segments and cards are lists, since YAML silently drops a repeated mapping key. A missing or unusable value is written as a blank card whose comment gives the reason. The InfluxDB settings check moves out of build_sink into a shared influx_settings() so the sink and the new source validate alike. Co-Authored-By: Claude Opus 5.5 --- docs/source/api/index.md | 32 +++ docs/source/index.md | 4 + docs/source/keyheader.md | 254 +++++++++++++++++++++++ libby/keygrabber/config.py | 58 +++--- libby/keyheader/__init__.py | 47 +++++ libby/keyheader/acquire.py | 114 +++++++++++ libby/keyheader/cards.py | 98 +++++++++ libby/keyheader/config.py | 357 +++++++++++++++++++++++++++++++++ libby/keyheader/daemon.py | 233 +++++++++++++++++++++ libby/keyheader/main.py | 45 +++++ pyproject.toml | 1 + tests/test_keyheader_config.py | 274 +++++++++++++++++++++++++ tests/test_keyheader_daemon.py | 262 ++++++++++++++++++++++++ tests/test_keyheader_values.py | 193 ++++++++++++++++++ 14 files changed, 1948 insertions(+), 24 deletions(-) create mode 100644 docs/source/keyheader.md create mode 100644 libby/keyheader/acquire.py create mode 100644 libby/keyheader/cards.py create mode 100644 libby/keyheader/config.py create mode 100644 libby/keyheader/daemon.py create mode 100644 libby/keyheader/main.py create mode 100644 tests/test_keyheader_config.py create mode 100644 tests/test_keyheader_daemon.py create mode 100644 tests/test_keyheader_values.py diff --git a/docs/source/api/index.md b/docs/source/api/index.md index 4277973..3bd014d 100644 --- a/docs/source/api/index.md +++ b/docs/source/api/index.md @@ -97,6 +97,38 @@ the docs build does not install the optional `influxdb` extra. :show-inheritance: ``` +## Keyheader + +```{eval-rst} +.. automodule:: libby.keyheader.source + :members: + :show-inheritance: +``` + +```{eval-rst} +.. automodule:: libby.keyheader.config + :members: + :show-inheritance: +``` + +```{eval-rst} +.. automodule:: libby.keyheader.acquire + :members: + :show-inheritance: +``` + +```{eval-rst} +.. automodule:: libby.keyheader.cards + :members: + :show-inheritance: +``` + +```{eval-rst} +.. automodule:: libby.keyheader.daemon + :members: + :show-inheritance: +``` + ## Telemetry ```{eval-rst} diff --git a/docs/source/index.md b/docs/source/index.md index 69db8e3..820eac2 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -13,6 +13,8 @@ with pluggable transports (ZMQ or RabbitMQ). It gives you: `wait_for` (libby's `ktl.waitFor`). - **Keygrabber**: a daemon that polls keywords from other peers and writes them to a time-series database for dashboarding. +- **Keyheader**: a daemon that answers FITS header segments for a moment from + the values the keygrabber stored. - **Telemetry**: optional OpenTelemetry traces and metrics for every request. - **`libby` CLI** — a command-line front end for keyword peers (`show` / `modify` / `list` / `describe` / `waitfor`), with TAB completion @@ -28,6 +30,7 @@ client cli daemon keygrabber +keyheader telemetry releasing api/index @@ -40,5 +43,6 @@ api/index - Writing a script or tool that talks to peers? Read {doc}`client`. - Poking at peers interactively? Read {doc}`cli`. - Recording keywords for Grafana? Read {doc}`keygrabber`. +- Writing instrument state into FITS headers? Read {doc}`keyheader`. - Tracing requests across peers? Read {doc}`telemetry`. - Looking for a specific class or function? See {doc}`api/index`. diff --git a/docs/source/keyheader.md b/docs/source/keyheader.md new file mode 100644 index 0000000..c1679ad --- /dev/null +++ b/docs/source/keyheader.md @@ -0,0 +1,254 @@ +# Keyheader + +Keyheader is a libby daemon that captures a snapshot of instrument state at a +moment and returns it as FITS header cards. A requesting daemon, usually a +camera daemon, asks for a named **segment** at a time such as the start of an +exposure, and adds the cards it gets back to its image header. + +The values come from the database the {doc}`keygrabber` fills, so keyheader +never talks to a hardware daemon and a slow one cannot delay a header. +InfluxDB 2.x is the only backend today: + +```bash +pip install libby[influxdb] +``` + +## Running it + +```bash +keyheader -c /etc/hispec/keyheader.yaml +``` + +Like the keygrabber, it is an ordinary `LibbyDaemon`: `SIGTERM` or the +`shutdown` keyword stops it, and it logs to stderr unless the config has a +`logging` section. + +## Config + +A keyheader config is an ordinary daemon config plus `source`, `defaults` and +`segments`. + +```yaml +peer_id: keyheader +group_id: hispec +transport: rabbitmq +rabbitmq_url: amqp://localhost + +source: + type: influxdb + url: http://influx.hispec:8086 + org: hispec + bucket: telemetry + token_env: HISPEC_INFLUX_TOKEN + +workers: 4 + +defaults: + mode: nearest + window_s: 300 + +segments: + - name: EXP-BEGIN + date_keyword: DATE-BEG + cards: + - fits: ADCPOS1 + keyword: hsfei.adc.positionvalue1 + type: float + format: ".3f" + comment: "ADC prism 1 position [deg]" + mode: interpolate + - fits: ADCMOVE + keyword: hsfei.adc.ismoving + type: bool + comment: "ADC in motion" + + - name: EXP-END + date_keyword: DATE-END + defaults: + mode: prior + cards: + - fits: ADCPOS1E + keyword: hsfei.adc.positionvalue1 + type: float + format: ".3f" + comment: "ADC prism 1 position at end [deg]" +``` + +### source + +Validated exactly like the keygrabber's `sink`: `type: influxdb`, and the token +is read from the environment variable `token_env` names. A `token` key in the +file is rejected. + +### segments + +`segments` is a list, and so is each segment's `cards`. They are not mappings +on purpose: YAML keeps only the last of two identical keys without warning, +so a duplicated FITS keyword written as a key would disappear rather than be +reported. A mapping is refused with an error that says so. + +A segment has: + +- `name`: letters, digits, `_`, `-` or `.`, at most 40 characters. Requests + use this name, and it starts the segment's COMMENT card. +- `date_keyword` (optional): one of `DATE-OBS`, `DATE-BEG`, `DATE-AVG` or + `DATE-END`. These are the FITS standard's own time keywords. The segment + writes its timestamp there as well as in the COMMENT. +- `date_comment` (optional): the comment for that card. +- `defaults` (optional): `mode` and `window_s` for this segment's cards. These + override the top-level `defaults`. +- `cards`: the mapped keywords, in the order they appear in the header. + +A card has: + +| Field | Required | Meaning | +|---|---|---| +| `fits` | yes | FITS keyword: 1-8 characters of `A-Z`, `0-9`, `-`, `_` | +| `keyword` | yes | Daemon keyword, `..` | +| `type` | yes | `float`, `int`, `bool` or `str` | +| `format` | no | Python format spec, e.g. `.3f`, `.2e`, `.20` | +| `comment` | no | Card comment, at most 47 printable ASCII characters | +| `mode` | no | `interpolate`, `nearest`, `prior` or `subsequent` | +| `window_s` | no | How far from the moment to look for a sample | + +### Validation + +Config load rejects: + +- a FITS keyword that breaks the naming rules +- a keyword the FITS 4.0 standard reserves. This covers the structural ones + (`SIMPLE`, `NAXISn`, `END`, ...), commentary (`COMMENT`, `HISTORY`, + `CONTINUE`), the general descriptive ones (`OBJECT`, `TELESCOP`, `DATE`, + ...), and the WCS and time keywords. Keyheader writes COMMENT itself. +- a FITS keyword used twice, either within one segment or across segments. + The requesting daemon merges segments into one header, and FITS forbids a + repeated keyword, so the error names both segments rather than renaming + either. +- a repeated segment name +- `interpolate` on a `bool` or `str` card +- a `format` the type cannot take, or a numeric format whose output does not + read back as a number (e.g. `,.2f`) +- a comment longer than 47 characters, the space left beside a fixed-format + value + +A bad config stops the daemon at startup, and a bad file given to `reload` +leaves the running segments in place. + +## Acquisition modes + +Every card is looked up independently, within `window_s` of the requested +moment: + +- `prior`: the latest sample at or before the moment +- `subsequent`: the earliest sample at or after the moment +- `nearest`: whichever of those two is closer. A tie goes to the earlier + sample, which was already true at the moment. +- `interpolate`: linear interpolation between the samples either side. When + only one side exists, it uses that value. + +That fallback matters at the end of an exposure. A request made then normally +arrives before the keygrabber has stored the next sample, and most values do +not change during an exposure, so the last stored value stands in. A +`subsequent` card has no such fallback and is written blank. + +`window_s` should exceed the keygrabber's interval for that keyword, including +its backoff, or a quiet keyword will come back blank. + +## Requesting a segment + +The service key is `keyheader.segment`: + +```python +reply = libby.rpc("hispec.keyheader", "keyheader.segment", + {"segment": "EXP-BEGIN", "time": "2026-11-11T08:15:02.123Z"}) +``` + +`time` is ISO 8601. A time with no offset is taken as UTC. The reply is: + +```json +{ + "ok": true, + "segment": "EXP-BEGIN", + "time": "2026-11-11T08:15:02.123", + "cards": [ + {"keyword": "COMMENT", "value": "EXP-BEGIN 2026-11-11T08:15:02.123", "comment": ""}, + {"keyword": "DATE-BEG", "value": "2026-11-11T08:15:02.123", "comment": "UTC time of this segment"}, + {"keyword": "ADCPOS1", "value": 12.346, "comment": "ADC prism 1 position [deg]"}, + {"keyword": "ADCMOVE", "value": false, "comment": "ADC in motion"} + ] +} +``` + +An unknown segment or a malformed request comes back as `ok: false` with an +`error`. + +### Adding the cards with astropy + +Keyheader does not write card images. The requesting daemon passes each entry +to astropy as a tuple: + +```python +cards = [(c["keyword"], c["value"], c["comment"]) for c in reply["cards"]] +header.extend(cards, end=True) +``` + +Use `end=True`, or build a fresh `fits.Header(cards)`. Plain +`header.extend(cards)` inserts keyword cards ahead of trailing commentary +cards, which moves the segment's COMMENT below the cards it introduces. + +The requesting daemon is still responsible for checking the final header +against the FITS standard. For example, it should not add a second segment +that names the same keywords. + +### Values and formats + +A value is typed JSON, and astropy decides the column layout. `format` +therefore sets precision rather than layout: the value is formatted and then +parsed back, so `.3f` turns `12.34567` into `12.346`. Other conversions: + +- An `int` card rounds, because InfluxDB stores every integer as a float. +- A `bool` card accepts a stored `0` or `1`. +- A `str` format can truncate, e.g. `.20`. + +### Blank cards + +A card is never left out. When there is no value, it is written with a FITS +undefined value (`null` in the reply, which astropy writes as a blank value), +and its comment starts with the reason: + +| Reason | Cause | +|---|---| +| `no sample`, `no prior sample`, `no later sample` | Nothing stored in the window | +| `read failed` | The database query failed | +| `not a float`, `not a bool`, ... | The stored value cannot be the configured type | +| `not finite` | NaN or infinity, which FITS cannot represent | +| `not numeric` | Interpolation between non-numbers | + +The reason and the original comment together are truncated to 47 characters. +One failed card never fails the request: losing one value should not cost the +exposure its whole header. + +### Latency + +The cards in a segment are read in parallel on `workers` threads. A request +therefore costs about one database round trip, plus queueing when a segment +has many more cards than workers. The request is handled on the transport's +receive thread, so it holds up other requests to keyheader for that long. +Give the RPC a TTL that covers the database's `timeout_ms`. + +## Control keywords + +| Keyword | Type | Access | Meaning | +|---|---|---|---| +| `isconnected` | bool | R | Last source read succeeded | +| `requests` | int | R | Segment requests answered since start | +| `readerrors` | int | R | Source reads that failed | +| `blankcards` | int | R | Cards answered with an undefined value | +| `segments` | string | R | Configured segment names, comma separated | +| `reload` | trigger | W | Re-read the config file and apply its segments | +| `shutdown` | trigger | W | Gracefully stop the daemon | + +`reload` can add, remove or change segments without a restart, because +segments are not keywords. A change to `source` or `workers` is refused with a +request to restart instead. A request already in progress finishes with the +segments it started with. \ No newline at end of file diff --git a/libby/keygrabber/config.py b/libby/keygrabber/config.py index 6d93cc1..408f12a 100644 --- a/libby/keygrabber/config.py +++ b/libby/keygrabber/config.py @@ -9,7 +9,7 @@ import os import re from dataclasses import dataclass -from typing import Any, List, Mapping, Sequence, Tuple +from typing import Any, Dict, List, Mapping, Sequence, Tuple from ..config import ConfigError from ..keyword import match_pattern @@ -88,43 +88,53 @@ def parse_config(config: Mapping[str, Any]) -> KeygrabberConfig: def build_sink(sink_config: Mapping[str, Any]) -> Sink: """Build the configured sink, importing its backend client on demand.""" - kind = str(sink_config.get("type", "")).strip().lower() + settings = influx_settings(sink_config, "sink") + # Deferred: influxdb-client is an optional extra, so a deployment using a + # different backend never has to install it + from .influx import ( # pylint: disable=import-outside-toplevel + InfluxConfig, InfluxSink, + ) + return InfluxSink(InfluxConfig(**settings)) + + +def influx_settings(section: Mapping[str, Any], label: str) -> Dict[str, Any]: + """Validate an InfluxDB config section into ``InfluxConfig`` arguments. + + Shared by the keygrabber's ``sink`` and keyheader's ``source``, which + describe the same database. Every field is checked before the backend is + imported, so a malformed config is reported the same way whether or not the + extra is installed. ``label`` names the section in error messages. + """ + kind = str(section.get("type", "")).strip().lower() if not kind: - raise ConfigError("sink.type is required") + raise ConfigError(f"{label}.type is required") if kind != "influxdb": - raise ConfigError(f"unsupported sink type {kind!r}; expected 'influxdb'") + raise ConfigError( + f"unsupported {label} type {kind!r}; expected 'influxdb'") - if "token" in sink_config: + if "token" in section: raise ConfigError( - "sink.token must not appear in the config; name an environment " - "variable with sink.token_env instead" + f"{label}.token must not appear in the config; name an environment " + f"variable with {label}.token_env instead" ) - token_env = _required(sink_config, "token_env", "sink") + token_env = _required(section, "token_env", label) token = os.environ.get(str(token_env)) if not token: raise ConfigError( f"environment variable {token_env} is unset or empty, so the " - "sink has no token" + f"{label} has no token" ) - # Validate every field before importing the backend, so a malformed config - # is reported the same way whether or not the extra is installed - settings = { - "url": str(_required(sink_config, "url", "sink")), - "org": str(_required(sink_config, "org", "sink")), - "bucket": str(_required(sink_config, "bucket", "sink")), + settings: Dict[str, Any] = { + "url": str(_required(section, "url", label)), + "org": str(_required(section, "org", label)), + "bucket": str(_required(section, "bucket", label)), "token": token, } - if "timeout_ms" in sink_config: + if "timeout_ms" in section: settings["timeout_ms"] = _positive_int( - sink_config["timeout_ms"], "sink.timeout_ms") - - # Deferred: influxdb-client is an optional extra, so a deployment using a - # different backend never has to install it - from .influx import ( # pylint: disable=import-outside-toplevel - InfluxConfig, InfluxSink, - ) - return InfluxSink(InfluxConfig(**settings)) + section["timeout_ms"], f"{label}.timeout_ms") + return settings def _parse_collection( diff --git a/libby/keyheader/__init__.py b/libby/keyheader/__init__.py index e69de29..cfc83ac 100644 --- a/libby/keyheader/__init__.py +++ b/libby/keyheader/__init__.py @@ -0,0 +1,47 @@ +"""Keyheader: answer FITS header segments from keyword values the keygrabber stored. + +A requesting daemon (usually a camera) asks for a named segment at a moment, +and gets back ``{keyword, value, comment}`` entries to add to its image header. + +Importing this package does not pull in any database client. ``InfluxSource`` +lives in :mod:`libby.keygrabber.influx` and needs the optional dependency +(``pip install libby[influxdb]``). +""" +from .acquire import Acquired, acquire, query_range +from .cards import card_entry, fits_date, segment_entries, to_fits_value +from .config import ( + CardConfig, + KeyheaderConfig, + Mode, + SegmentConfig, + ValueType, + build_source, + is_reserved, + parse_config, +) +from .daemon import SERVICE_KEY, KeyheaderDaemon, parse_time +from .source import Source, SourceError, SourceReadError + +__all__ = [ + "Acquired", + "CardConfig", + "KeyheaderConfig", + "KeyheaderDaemon", + "Mode", + "SERVICE_KEY", + "SegmentConfig", + "Source", + "SourceError", + "SourceReadError", + "ValueType", + "acquire", + "build_source", + "card_entry", + "fits_date", + "is_reserved", + "parse_config", + "parse_time", + "query_range", + "segment_entries", + "to_fits_value", +] diff --git a/libby/keyheader/acquire.py b/libby/keyheader/acquire.py new file mode 100644 index 0000000..28632ce --- /dev/null +++ b/libby/keyheader/acquire.py @@ -0,0 +1,114 @@ +"""Choose a keyword's value at one moment from the samples stored around it. + +Pure functions over already-fetched samples, so every acquisition mode can be +tested without a database. :func:`query_range` says what to fetch and +:func:`acquire` picks from it. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from numbers import Real +from typing import Optional, Sequence, Tuple + +from ..keygrabber.sink import Sample, Value +from .config import Mode + +# A source reads ``[start, stop)``, so a sample stamped exactly at the +# requested moment needs the range to run just past it +_INCLUSIVE = timedelta(microseconds=1) + + +@dataclass(frozen=True) +class Acquired: + """A chosen value, or the reason there is none. + + ``reason`` is set exactly when ``value`` is None, and is short enough to + put on the card in place of the value. + """ + + value: Value + reason: Optional[str] = None + + @classmethod + def missing(cls, reason: str) -> "Acquired": + """Return a result with no value, saying why.""" + return cls(None, reason) + + +def query_range(moment: datetime, mode: Mode, + window_s: float) -> Tuple[datetime, datetime]: + """Return the ``[start, stop)`` range a mode needs samples from.""" + window = timedelta(seconds=window_s) + if mode is Mode.PRIOR: + return moment - window, moment + _INCLUSIVE + if mode is Mode.SUBSEQUENT: + return moment, moment + window + return moment - window, moment + window + + +def acquire(samples: Sequence[Sample], moment: datetime, mode: Mode) -> Acquired: + """Pick the value ``mode`` asks for from samples sorted oldest first. + + Null values are ignored, since they say nothing about the keyword's state. + """ + usable = [sample for sample in samples if sample.value is not None] + before = _last(sample for sample in usable if sample.timestamp <= moment) + after = _first(sample for sample in usable if sample.timestamp >= moment) + + if mode is Mode.PRIOR: + return _from(before, "no prior sample") + if mode is Mode.SUBSEQUENT: + return _from(after, "no later sample") + if mode is Mode.NEAREST: + return _from(_closer(before, after, moment), "no sample") + return _interpolate(before, after, moment) + + +def _interpolate(before: Optional[Sample], after: Optional[Sample], + moment: datetime) -> Acquired: + """Interpolate linearly, or fall back to the one side that exists. + + A request made at the end of an exposure usually comes before the next + sample is stored. Most values are static across an exposure, so the last + stored one stands in rather than leaving the card blank. + """ + if before is None or after is None: + return _from(before or after, "no sample") + if before.timestamp == after.timestamp: + return Acquired(before.value) + if not (_is_number(before.value) and _is_number(after.value)): + return Acquired.missing("not numeric") + span = (after.timestamp - before.timestamp).total_seconds() + fraction = (moment - before.timestamp).total_seconds() / span + low, high = float(before.value), float(after.value) # type: ignore[arg-type] + return Acquired(low + (high - low) * fraction) + + +def _closer(before: Optional[Sample], after: Optional[Sample], + moment: datetime) -> Optional[Sample]: + """Return whichever sample is nearer in time, preferring the earlier on a tie.""" + if before is None or after is None: + return before or after + if (after.timestamp - moment) < (moment - before.timestamp): + return after + return before + + +def _from(sample: Optional[Sample], reason: str) -> Acquired: + return Acquired.missing(reason) if sample is None else Acquired(sample.value) + + +def _is_number(value: Value) -> bool: + return isinstance(value, Real) and not isinstance(value, bool) + + +def _first(samples) -> Optional[Sample]: + return next(iter(samples), None) + + +def _last(samples) -> Optional[Sample]: + found = None + for found in samples: + pass + return found diff --git a/libby/keyheader/cards.py b/libby/keyheader/cards.py new file mode 100644 index 0000000..b48de85 --- /dev/null +++ b/libby/keyheader/cards.py @@ -0,0 +1,98 @@ +"""Turn acquired values into FITS card entries. + +An entry is ``{"keyword", "value", "comment"}`` with a JSON value, which the +requesting daemon hands to astropy as a ``(keyword, value, comment)`` tuple. +Keyheader leaves the 80-column layout to astropy rather than writing card +images itself, so the format spec controls a value's precision, not its +column placement. + +A value that is missing or unusable becomes ``None``, which astropy writes as +a FITS undefined value: the card is still present, and its comment says why +it is blank. +""" +from __future__ import annotations + +import math +from datetime import datetime, timezone +from typing import Any, Dict, List + +from ..keygrabber.sink import Value +from .acquire import Acquired +from .config import MAX_COMMENT, CardConfig, SegmentConfig, ValueType + +Entry = Dict[str, Any] + + +class ValueTypeError(ValueError): + """A stored value cannot be written as the card's configured type.""" + + +def fits_date(moment: datetime) -> str: + """Format a moment as a FITS date string in UTC, to the millisecond.""" + utc = moment.astimezone(timezone.utc) + return utc.strftime("%Y-%m-%dT%H:%M:%S.") + f"{utc.microsecond // 1000:03d}" + + +def segment_entries(segment: SegmentConfig, moment: datetime, + acquired: List[Acquired]) -> List[Entry]: + """Build a segment's entries: its COMMENT, its date keyword, then its cards. + + ``acquired`` holds one result per card, in the segment's card order. + """ + stamp = fits_date(moment) + entries: List[Entry] = [entry("COMMENT", f"{segment.name} {stamp}", "")] + if segment.date_keyword is not None: + entries.append(entry(segment.date_keyword, stamp, segment.date_comment)) + entries.extend(card_entry(card, result) + for card, result in zip(segment.cards, acquired)) + return entries + + +def card_entry(card: CardConfig, acquired: Acquired) -> Entry: + """Build one card, blank with an explanatory comment if there is no value.""" + if acquired.reason is not None: + return _blank(card, acquired.reason) + try: + return entry(card.fits, to_fits_value(acquired.value, card), card.comment) + except ValueTypeError as exc: + return _blank(card, str(exc)) + + +def to_fits_value(value: Value, card: CardConfig) -> Value: + """Coerce a stored value to the card's type, at the format's precision. + + The format is applied and the result parsed back, so a ``.3f`` float + carries three decimals of precision while staying a number astropy writes + as one. InfluxDB stores every integer as a float, so an int card rounds. + """ + kind = card.type + if kind is ValueType.BOOL: + if isinstance(value, bool): + return value + if isinstance(value, (int, float)) and value in (0, 1): + return bool(value) + raise ValueTypeError("not a bool") + if kind is ValueType.STR: + text = str(value) + return format(text, card.format) if card.format else text + + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueTypeError(f"not a {kind.value}") + if not math.isfinite(value): + raise ValueTypeError("not finite") + if kind is ValueType.INT: + number = int(round(value)) + return int(format(number, card.format)) if card.format else number + number = float(value) + return float(format(number, card.format)) if card.format else number + + +def entry(keyword: str, value: Value, comment: str) -> Entry: + """Return one entry in the shape the requesting daemon expects.""" + return {"keyword": keyword, "value": value, "comment": comment} + + +def _blank(card: CardConfig, reason: str) -> Entry: + """A card with an undefined value, whose comment leads with the reason.""" + comment = f"{reason}: {card.comment}" if card.comment else reason + return entry(card.fits, None, comment[:MAX_COMMENT]) diff --git a/libby/keyheader/config.py b/libby/keyheader/config.py new file mode 100644 index 0000000..b7e16da --- /dev/null +++ b/libby/keyheader/config.py @@ -0,0 +1,357 @@ +"""Parse and validate keyheader's own configuration sections. + +A keyheader config file is an ordinary ``LibbyDaemon`` config (``peer_id``, +``group_id``, ``transport``, ...) plus three sections this module owns: +``source``, ``defaults`` and ``segments``. + +Segments and their cards are lists rather than mappings on purpose. YAML keeps +only the last of two identical mapping keys without a word, so a duplicated +FITS keyword or segment name written as a key would vanish instead of being +reported. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, Mapping, Optional, Sequence, Tuple + +from ..config import ConfigError +# The keygrabber's validators, so both daemons phrase config errors alike +from ..keygrabber.config import ( + _mapping, _positive_float, _positive_int, _required, influx_settings, +) +from .source import Source + +DEFAULT_WINDOW_S = 300.0 +DEFAULT_WORKERS = 4 +DEFAULT_DATE_COMMENT = "UTC time of this segment" + +# Value and comment share the 70 columns after ``KEYWORD= ``. A fixed-format +# value takes 20 of them and the ``/`` separator 3, which leaves this much for +# a comment that is never truncated. +MAX_COMMENT = 47 + +# A COMMENT card's text runs from column 9 to 80, and carries the segment name +# and a timestamp of up to 23 characters +MAX_SEGMENT_NAME = 40 + +_FITS_KEYWORD = re.compile(r"^[A-Z0-9_-]{1,8}$") +_SEGMENT_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") + +# Keywords the FITS 4.0 standard reserves (sections 4.4, 7, 8 and 9), as +# patterns: ``n``/``i``/``j``/``m`` are axis or field numbers and a trailing +# ``a`` is an alternate WCS letter. A mapped keyword may not take any of them, +# because the standard gives each one a meaning or a structural role, and +# COMMENT and HISTORY are written by keyheader itself. +_A = "[A-Z]?" +_N = "[0-9]+" +RESERVED_PATTERNS: Tuple[str, ...] = ( + # Mandatory and structural + "SIMPLE", "BITPIX", "NAXIS", f"NAXIS{_N}", "END", "XTENSION", "PCOUNT", + "GCOUNT", "EXTEND", "GROUPS", f"PTYPE{_N}", f"PSCAL{_N}", f"PZERO{_N}", + "TFIELDS", f"TFORM{_N}", f"TTYPE{_N}", f"TUNIT{_N}", f"TSCAL{_N}", + f"TZERO{_N}", f"TNULL{_N}", f"TDISP{_N}", f"TDIM{_N}", f"TBCOL{_N}", + "THEAP", f"TDMIN{_N}", f"TDMAX{_N}", f"TLMIN{_N}", f"TLMAX{_N}", + # Commentary + "COMMENT", "HISTORY", "CONTINUE", + # General descriptive + "DATE", "ORIGIN", "BLOCKED", "AUTHOR", "REFERENC", "OBSERVER", "TELESCOP", + "INSTRUME", "OBJECT", "EPOCH", "BUNIT", "BSCALE", "BZERO", "BLANK", + "DATAMAX", "DATAMIN", "EXTNAME", "EXTVER", "EXTLEVEL", "INHERIT", + "CHECKSUM", "DATASUM", + # World coordinates + f"WCSAXES{_A}", f"WCSNAME{_A}", f"CTYPE{_N}{_A}", f"CUNIT{_N}{_A}", + f"CRPIX{_N}{_A}", f"CRVAL{_N}{_A}", f"CDELT{_N}{_A}", f"CROTA{_N}", + f"CNAME{_N}{_A}", f"CRDER{_N}{_A}", f"CSYER{_N}{_A}", + f"PC{_N}_{_N}{_A}", f"CD{_N}_{_N}{_A}", f"PV{_N}_{_N}{_A}", + f"PS{_N}_{_N}{_A}", f"LONPOLE{_A}", f"LATPOLE{_A}", f"RADESYS{_A}", + "RADECSYS", f"EQUINOX{_A}", f"SPECSYS{_A}", f"SSYSOBS{_A}", + f"SSYSSRC{_A}", f"VELOSYS{_A}", f"ZSOURCE{_A}", f"VELANGL{_A}", + f"RESTFRQ{_A}", f"RESTWAV{_A}", "RESTFREQ", "OBSGEO-[XYZBLH]", + # Time + "DATE-OBS", "DATE-BEG", "DATE-AVG", "DATE-END", "MJD-OBS", "MJD-BEG", + "MJD-AVG", "MJD-END", "TIMESYS", "MJDREF", "MJDREFI", "MJDREFF", "JDREF", + "JDREFI", "JDREFF", "DATEREF", "TREFPOS", "TREFDIR", "PLEPHEM", + "TIMEUNIT", "TIMEOFFS", "TSTART", "TSTOP", "TIMSYER", "TIMRDER", + "TIMEDEL", "TIMEPIXR", "TELAPSE", "XPOSURE", +) +_RESERVED = re.compile("^(?:" + "|".join(RESERVED_PATTERNS) + ")$") + +# The time keywords a segment may name for its own timestamp. They are +# reserved for exactly this meaning, so a segment can use them where a mapped +# keyword cannot. +DATE_KEYWORDS: Tuple[str, ...] = ("DATE-OBS", "DATE-BEG", "DATE-AVG", "DATE-END") + + +class Mode(str, Enum): + """How a value is chosen from the samples around the requested time.""" + + INTERPOLATE = "interpolate" + NEAREST = "nearest" + PRIOR = "prior" + SUBSEQUENT = "subsequent" + + +class ValueType(str, Enum): + """The FITS value type a card is written with.""" + + FLOAT = "float" + INT = "int" + BOOL = "bool" + STR = "str" + + +_EXAMPLE_VALUES: Dict[ValueType, Any] = { + ValueType.FLOAT: 1234.5, ValueType.INT: 1234, ValueType.STR: "text", +} + + +# A config record, so the field count is the schema rather than complexity +@dataclass(frozen=True) +class CardConfig: # pylint: disable=too-many-instance-attributes + """One FITS keyword and the daemon keyword its value comes from.""" + + fits: str + group: str + peer: str + keyword: str + type: ValueType + format: Optional[str] + comment: str + mode: Mode + window_s: float + + @property + def source_name(self) -> str: + """Return the qualified daemon keyword, as written in the config.""" + return f"{self.group}.{self.peer}.{self.keyword}" + + +@dataclass(frozen=True) +class SegmentConfig: + """A named set of cards, requested together for one moment.""" + + name: str + date_keyword: Optional[str] + date_comment: str + cards: Tuple[CardConfig, ...] + + +@dataclass(frozen=True) +class KeyheaderConfig: + """Keyheader's parsed configuration.""" + + segments: Tuple[SegmentConfig, ...] + source: Mapping[str, Any] + workers: int + + def segment(self, name: str) -> SegmentConfig: + """Return a segment by name, or raise naming the ones that exist.""" + for segment in self.segments: + if segment.name == name: + return segment + known = ", ".join(segment.name for segment in self.segments) + raise ValueError(f"unknown segment {name!r}; configured: {known}") + + +def parse_config(config: Mapping[str, Any]) -> KeyheaderConfig: + """Validate a daemon config's keyheader sections into typed objects.""" + defaults = _mapping(config, "defaults") + raw_segments = config.get("segments") + if isinstance(raw_segments, Mapping): + raise ConfigError( + "segments must be a list of entries with a name, not a mapping: " + "YAML silently drops a repeated mapping key, which would hide a " + "duplicate segment") + if not isinstance(raw_segments, Sequence) or isinstance(raw_segments, str) \ + or not raw_segments: + raise ConfigError("segments must be a non-empty list") + + segments = tuple(_parse_segment(index, raw, defaults) + for index, raw in enumerate(raw_segments)) + _check_unique(segments) + return KeyheaderConfig( + segments=segments, + source=_mapping(config, "source"), + workers=_positive_int(config.get("workers", DEFAULT_WORKERS), "workers"), + ) + + +def build_source(source_config: Mapping[str, Any]) -> Source: + """Build the configured source, importing its backend client on demand.""" + settings = influx_settings(source_config, "source") + # Deferred: influxdb-client is an optional extra + from ..keygrabber.influx import ( # pylint: disable=import-outside-toplevel + InfluxConfig, InfluxSource, + ) + return InfluxSource(InfluxConfig(**settings)) + + +def is_reserved(keyword: str) -> bool: + """Return whether the FITS standard reserves this keyword.""" + return bool(_RESERVED.match(keyword)) + + +def _parse_segment(index: int, raw: Any, + defaults: Mapping[str, Any]) -> SegmentConfig: + if not isinstance(raw, Mapping): + raise ConfigError(f"segments[{index}] must be a mapping") + name = str(_required(raw, "name", f"segments[{index}]")) + if not _SEGMENT_NAME.match(name) or len(name) > MAX_SEGMENT_NAME: + raise ConfigError( + f"segment name {name!r} must be letters, digits, '_', '-' or '.', " + f"at most {MAX_SEGMENT_NAME} characters, so it fits a COMMENT card") + label = f"segment {name!r}" + + date_keyword = raw.get("date_keyword") + if date_keyword is not None: + date_keyword = str(date_keyword) + if date_keyword not in DATE_KEYWORDS: + raise ConfigError( + f"{label} date_keyword {date_keyword!r} must be one of " + f"{', '.join(DATE_KEYWORDS)}") + date_comment = _comment( + raw.get("date_comment", DEFAULT_DATE_COMMENT), f"{label} date_comment") + + raw_cards = raw.get("cards") + if isinstance(raw_cards, Mapping): + raise ConfigError( + f"{label} cards must be a list of entries with a fits name, not a " + "mapping: YAML silently drops a repeated mapping key, which would " + "hide a duplicate keyword") + if not isinstance(raw_cards, Sequence) or isinstance(raw_cards, str) \ + or not raw_cards: + raise ConfigError(f"{label} cards must be a non-empty list") + + defaults = {**defaults, **_mapping(raw, "defaults")} + cards = tuple(_parse_card(f"{label} cards[{position}]", card, defaults) + for position, card in enumerate(raw_cards)) + return SegmentConfig(name=name, date_keyword=date_keyword, + date_comment=date_comment, cards=cards) + + +def _parse_card(label: str, raw: Any, defaults: Mapping[str, Any]) -> CardConfig: + if not isinstance(raw, Mapping): + raise ConfigError(f"{label} must be a mapping") + + fits = str(_required(raw, "fits", label)) + if not _FITS_KEYWORD.match(fits): + raise ConfigError( + f"{label} FITS keyword {fits!r} must be 1-8 characters of A-Z, " + "0-9, '-' or '_'") + label = f"{label} ({fits})" + if is_reserved(fits): + raise ConfigError( + f"{label} is reserved by the FITS standard and cannot be mapped") + + group, peer, keyword = _parse_source_name( + label, _required(raw, "keyword", label)) + value_type = _enum(ValueType, _required(raw, "type", label), f"{label} type") + mode = _enum(Mode, raw.get("mode", defaults.get("mode", Mode.NEAREST.value)), + f"{label} mode") + if mode is Mode.INTERPOLATE and value_type in (ValueType.BOOL, ValueType.STR): + raise ConfigError( + f"{label} cannot interpolate a {value_type.value} value; use " + "nearest, prior or subsequent") + + return CardConfig( + fits=fits, + group=group, + peer=peer, + keyword=keyword, + type=value_type, + format=_format(raw.get("format"), value_type, label), + comment=_comment(raw.get("comment", ""), f"{label} comment"), + mode=mode, + window_s=_positive_float( + raw.get("window_s", defaults.get("window_s", DEFAULT_WINDOW_S)), + f"{label} window_s"), + ) + + +def _check_unique(segments: Sequence[SegmentConfig]) -> None: + """Reject a repeated segment name, or a FITS keyword used twice anywhere. + + Keywords are unique across every segment, not just within one, because the + requesting daemon merges segments into one header and FITS forbids a + repeated keyword. + """ + seen_segments: Dict[str, int] = {} + for index, segment in enumerate(segments): + if segment.name in seen_segments: + raise ConfigError( + f"segment name {segment.name!r} appears twice " + f"(segments[{seen_segments[segment.name]}] and " + f"segments[{index}])") + seen_segments[segment.name] = index + + owners: Dict[str, str] = {} + for segment in segments: + names = [card.fits for card in segment.cards] + if segment.date_keyword is not None: + names.insert(0, segment.date_keyword) + for name in names: + if name in owners: + where = ("twice in segment " + repr(segment.name) + if owners[name] == segment.name + else f"in segments {owners[name]!r} and " + f"{segment.name!r}") + raise ConfigError(f"FITS keyword {name} appears {where}") + owners[name] = segment.name + + +def _parse_source_name(label: str, raw: Any) -> Tuple[str, str, str]: + """Split ``..`` into its parts.""" + text = str(raw) + parts = text.split(".", 2) + if len(parts) != 3 or not all(parts) or "%" in text: + raise ConfigError( + f"{label} keyword {text!r} must be '..'") + return parts[0], parts[1], parts[2] + + +def _format(raw: Any, value_type: ValueType, label: str) -> Optional[str]: + """Validate a Python format spec against the card's type.""" + if raw is None: + return None + spec = str(raw) + if value_type is ValueType.BOOL: + raise ConfigError(f"{label} format does not apply to a bool, which " + "FITS always writes as T or F") + try: + text = format(_EXAMPLE_VALUES[value_type], spec) + except (ValueError, TypeError) as exc: + raise ConfigError( + f"{label} format {spec!r} is not valid for a " + f"{value_type.value}: {exc}") from exc + # A number is formatted and parsed back to set its precision, so a spec + # that adds separators or padding text would fail on every request + if value_type is not ValueType.STR: + try: + (float if value_type is ValueType.FLOAT else int)(text) + except ValueError: + raise ConfigError( + f"{label} format {spec!r} gives {text!r}, which does not read " + f"back as a {value_type.value}") from None + return spec + + +def _comment(raw: Any, label: str) -> str: + text = "" if raw is None else str(raw) + if len(text) > MAX_COMMENT: + raise ConfigError( + f"{label} is {len(text)} characters; at most {MAX_COMMENT} fit on " + "the card without truncation") + if not text.isascii() or not text.isprintable(): + raise ConfigError(f"{label} must be printable ASCII") + return text + + +def _enum(kind: Any, raw: Any, label: str) -> Any: + try: + return kind(str(raw).strip().lower()) + except ValueError: + choices = ", ".join(member.value for member in kind) + raise ConfigError( + f"{label} {raw!r} must be one of {choices}") from None diff --git a/libby/keyheader/daemon.py b/libby/keyheader/daemon.py new file mode 100644 index 0000000..f43741a --- /dev/null +++ b/libby/keyheader/daemon.py @@ -0,0 +1,233 @@ +"""The keyheader daemon: answer a named segment at a moment with FITS cards.""" +from __future__ import annotations + +import threading +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from typing import Any, List, Mapping, Optional + +from ..config import ConfigError, DaemonConfigLoader, with_env_overrides +from ..daemon import LibbyDaemon +from ..errors import LibbyError +from .acquire import Acquired, acquire, query_range +from .cards import Entry, fits_date, segment_entries +from .config import CardConfig, KeyheaderConfig, build_source, parse_config +from .source import Source, SourceError + +SERVICE_KEY = "keyheader.segment" + + +class Counters: # pylint: disable=too-few-public-methods + """Tallies the daemon reports, guarded for cross-thread increments.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self.requests = 0 + self.read_errors = 0 + self.blank_cards = 0 + + def add(self, **deltas: int) -> None: + """Add to one or more counters.""" + with self._lock: + for name, delta in deltas.items(): + setattr(self, name, getattr(self, name) + delta) + + +# Coordinates config, source, pool and reload state; the attribute count is +# collaborators, not state +class KeyheaderDaemon(LibbyDaemon): # pylint: disable=too-many-instance-attributes + """Answers ``keyheader.segment`` requests with FITS card entries. + + A request names a configured segment and a moment. Every card in the + segment is looked up in the source at that moment, and the reply is a list + of ``{keyword, value, comment}`` entries the requesting daemon adds to its + image header. + """ + + transport = "rabbitmq" + discovery_enabled = False + + def __init__(self) -> None: + super().__init__() + self.counters = Counters() + self._settings: Optional[KeyheaderConfig] = None + self._source: Optional[Source] = None + self._pool: Optional[ThreadPoolExecutor] = None + self._source_healthy = True + self._config_path: Optional[str] = None + self._daemon_id: Optional[str] = None + self._env_prefix: Optional[str] = None + + def on_start(self) -> None: + """Parse config, open the source, and start answering requests.""" + self.configure(parse_config(self._config)) + self.add_service(SERVICE_KEY, self._serve) + self._register_control_keywords() + self.logger.info("serving %d segments: %s", len(self._require().segments), + ", ".join(s.name for s in self._require().segments)) + + def on_stop(self) -> None: + """Stop the query pool and release the source.""" + pool, self._pool = self._pool, None + if pool is not None: + pool.shutdown(wait=True) + source, self._source = self._source, None + if source is not None: + source.close() + + def configure(self, settings: KeyheaderConfig, + source: Optional[Source] = None) -> None: + """Apply parsed settings and open the source. + + Called by ``on_start``; a test calls it directly with a fake source to + exercise :meth:`segment` without a transport or a database. + """ + self._settings = settings + self._source = source if source is not None else self.make_source() + self._source.connect() + self._pool = ThreadPoolExecutor(max_workers=settings.workers, + thread_name_prefix="keyheader-read") + + @classmethod + def from_config_file(cls, path, daemon_id=None, *, env_prefix=None): + """Build from a file, remembering where it came from for ``reload``.""" + daemon = super().from_config_file(path, daemon_id, env_prefix=env_prefix) + daemon._config_path = str(path) + daemon._daemon_id = daemon_id + daemon._env_prefix = env_prefix + return daemon + + def make_source(self) -> Source: + """Build the configured source. + + Override to supply a source the config cannot describe, or to inject + one in a test without installing a database client. + """ + return build_source(self._require().source) + + ### Requests + + def segment(self, name: str, moment: datetime) -> List[Entry]: + """Return the entries for one segment at one moment. + + Each card is read on the pool, so a segment costs about one query's + latency rather than one per card. A card whose read fails is written + blank rather than failing the request: one missing value should not + cost the exposure its whole header. + """ + settings = self._require() + segment = settings.segment(name) + pool = self._pool + if pool is None: + raise LibbyError("keyheader is not started") + results = list(pool.map(lambda card: self._acquire(card, moment), + segment.cards)) + self.counters.add( + requests=1, + blank_cards=sum(1 for result in results if result.reason)) + return segment_entries(segment, moment, results) + + def _serve(self, payload: Mapping[str, Any]) -> dict: + """Handle a ``keyheader.segment`` request.""" + name = payload.get("segment") + if not isinstance(name, str) or not name: + raise ValueError("request needs a 'segment' name") + moment = parse_time(payload.get("time")) + entries = self.segment(name, moment) + return {"ok": True, "segment": name, "time": fits_date(moment), + "cards": entries} + + def _acquire(self, card: CardConfig, moment: datetime) -> Acquired: + source = self._source + if source is None: + return Acquired.missing("not started") + start, stop = query_range(moment, card.mode, card.window_s) + try: + samples = source.read(card.keyword, start, stop, + group=card.group, peer=card.peer) + except SourceError as exc: + self.counters.add(read_errors=1) + self._note_source(False, str(exc)) + return Acquired.missing("read failed") + self._note_source(True) + return acquire(samples, moment, card.mode) + + def _note_source(self, healthy: bool, reason: str = "") -> None: + """Track source health, logging only when it changes.""" + if healthy == self._source_healthy: + return + self._source_healthy = healthy + if healthy: + self.logger.info("source reads are succeeding again") + else: + self.logger.error("source read failed: %s", reason) + + ### Control surface + + def _register_control_keywords(self) -> None: + registry = self.keyword_registry + registry.bool("isconnected", getter=lambda: self._source_healthy, + description="Last source read succeeded.") + registry.int("requests", getter=lambda: self.counters.requests, + description="Segment requests answered since start.") + registry.int("readerrors", getter=lambda: self.counters.read_errors, + description="Source reads that failed.") + registry.int("blankcards", getter=lambda: self.counters.blank_cards, + description="Cards answered with an undefined value.") + registry.string("segments", getter=self._segment_names, + description="Configured segment names, comma separated.") + registry.trigger("reload", action=self._reload, + description="Re-read the config file and apply its " + "segments.") + registry.trigger("shutdown", action=self.request_stop, + description="Gracefully stop this daemon.") + + def _segment_names(self) -> str: + settings = self._settings + return ",".join(s.name for s in settings.segments) if settings else "" + + def _reload(self) -> None: + """Re-read the config file and swap in its segments. + + Segments are not keywords, so adding or removing them needs no restart. + A change to ``source`` or ``workers`` does, and is refused rather than + half-applied. A request in flight keeps the settings it started with. + """ + if self._config_path is None: + raise LibbyError("this keyheader was not built from a config file") + config = DaemonConfigLoader(self._config_path).get_daemon_config( + self._daemon_id) + if self._env_prefix: + config = with_env_overrides(config, prefix=self._env_prefix) + settings = parse_config(config) + running = self._require() + if (dict(settings.source) != dict(running.source) + or settings.workers != running.workers): + raise ConfigError("reload cannot change source or workers; " + "restart instead") + self._settings = settings + self.logger.info("reloaded %d segments from %s", + len(settings.segments), self._config_path) + + def _require(self) -> KeyheaderConfig: + if self._settings is None: + raise LibbyError("keyheader config has not been parsed yet") + return self._settings + + +def parse_time(raw: Any) -> datetime: + """Parse a request's ISO 8601 time; a time with no offset is taken as UTC.""" + if not isinstance(raw, str) or not raw.strip(): + raise ValueError("request needs a 'time' in ISO 8601, e.g. " + "'2026-11-11T08:15:02.123Z'") + text = raw.strip() + # fromisoformat accepts a trailing Z only from Python 3.11 + if text.endswith(("Z", "z")): + text = text[:-1] + "+00:00" + try: + moment = datetime.fromisoformat(text) + except ValueError: + raise ValueError(f"time {raw!r} is not ISO 8601") from None + if moment.tzinfo is None: + moment = moment.replace(tzinfo=timezone.utc) + return moment diff --git a/libby/keyheader/main.py b/libby/keyheader/main.py new file mode 100644 index 0000000..58bdb8e --- /dev/null +++ b/libby/keyheader/main.py @@ -0,0 +1,45 @@ +"""Console entry point for the keyheader daemon. + +Exits live here rather than in the daemon, so the daemon stays importable and +raises catchable exceptions. +""" +from __future__ import annotations + +import argparse +import sys +from collections.abc import Mapping +from typing import List, Optional + +from ..errors import LibbyError +from ..keygrabber.main import log_to_stderr +from .daemon import KeyheaderDaemon + + +def main(argv: Optional[List[str]] = None) -> int: + """Run keyheader until it is stopped. Return an exit code.""" + parser = argparse.ArgumentParser( + prog="keyheader", + description="Answer FITS header segments from stored keyword values", + ) + parser.add_argument("-c", "--config", required=True, + help="path to the keyheader config (YAML or JSON)") + parser.add_argument("-d", "--daemon-id", default=None, + help="daemon id, for a subsystem config with several") + namespace = parser.parse_args(argv) + + try: + daemon = KeyheaderDaemon.from_config_file(namespace.config, + namespace.daemon_id) + if not isinstance(daemon.get_config("logging"), Mapping): + log_to_stderr() + daemon.serve() + except KeyboardInterrupt: + return 0 + except (LibbyError, OSError, ValueError) as exc: + print(f"keyheader: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 6ea2c92..5535c41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,3 +64,4 @@ exclude = ["peers*", "package*"] [project.scripts] libby = "libby.cli.libby_cli:main" keygrabber = "libby.keygrabber.main:main" +keyheader = "libby.keyheader.main:main" diff --git a/tests/test_keyheader_config.py b/tests/test_keyheader_config.py new file mode 100644 index 0000000..1515c28 --- /dev/null +++ b/tests/test_keyheader_config.py @@ -0,0 +1,274 @@ +"""Unit tests for keyheader config parsing and FITS keyword validation.""" +from __future__ import annotations + +import copy +import os +import unittest +from typing import Any, Dict +from unittest import mock + +import yaml + +from libby.config import ConfigError +from libby.keyheader import Mode, ValueType, is_reserved, parse_config +from libby.keyheader.config import DEFAULT_WINDOW_S, MAX_COMMENT, build_source + +BASE: Dict[str, Any] = { + "peer_id": "keyheader", + "group_id": "hispec", + "defaults": {"mode": "nearest", "window_s": 120}, + "segments": [ + { + "name": "EXP-BEGIN", + "date_keyword": "DATE-BEG", + "cards": [ + {"fits": "ADCPOS1", "keyword": "hsfei.adc.positionvalue1", + "type": "float", "format": ".3f", "comment": "ADC position", + "mode": "interpolate"}, + {"fits": "ADCMOVE", "keyword": "hsfei.adc.ismoving", + "type": "bool"}, + ], + }, + { + "name": "EXP-END", + "date_keyword": "DATE-END", + "cards": [ + {"fits": "ADCPOS1E", "keyword": "hsfei.adc.positionvalue1", + "type": "float", "mode": "prior", "window_s": 30}, + ], + }, + ], +} + + +def _config(**changes: Any) -> Dict[str, Any]: + config = copy.deepcopy(BASE) + config.update(changes) + return config + + +def _with_card(**fields: Any) -> Dict[str, Any]: + """Return the base config with one more card in EXP-BEGIN.""" + config = _config() + card = {"fits": "EXTRA", "keyword": "hsfei.adc.x", "type": "float"} + card.update(fields) + config["segments"][0]["cards"].append(card) + return config + + +class ParseTests(unittest.TestCase): + """A valid config parses into typed segments and cards.""" + + def test_segments_and_cards_keep_config_order(self): + """Order matters: it is the order the cards appear in the header.""" + settings = parse_config(_config()) + self.assertEqual([s.name for s in settings.segments], + ["EXP-BEGIN", "EXP-END"]) + self.assertEqual([c.fits for c in settings.segments[0].cards], + ["ADCPOS1", "ADCMOVE"]) + + def test_card_fields_are_typed(self): + """Split the source keyword and convert type and mode to enums.""" + card = parse_config(_config()).segments[0].cards[0] + self.assertEqual((card.group, card.peer, card.keyword), + ("hsfei", "adc", "positionvalue1")) + self.assertIs(card.type, ValueType.FLOAT) + self.assertIs(card.mode, Mode.INTERPOLATE) + self.assertEqual(card.format, ".3f") + + def test_defaults_and_overrides(self): + """Fall back to defaults, and let a card override them.""" + settings = parse_config(_config()) + moving = settings.segments[0].cards[1] + self.assertIs(moving.mode, Mode.NEAREST) + self.assertEqual(moving.window_s, 120.0) + self.assertEqual(settings.segments[1].cards[0].window_s, 30.0) + + def test_builtin_default_window(self): + """Use the built-in window when neither defaults nor the card set one.""" + config = _config(defaults={}) + card = parse_config(config).segments[0].cards[1] + self.assertEqual(card.window_s, DEFAULT_WINDOW_S) + + def test_segment_defaults_override_global(self): + """Let a segment set defaults for its own cards.""" + config = _config() + config["segments"][1]["defaults"] = {"mode": "prior"} + del config["segments"][1]["cards"][0]["mode"] + self.assertIs(parse_config(config).segments[1].cards[0].mode, Mode.PRIOR) + + def test_lookup_by_name(self): + """Find a segment by name, and name the known ones when missing.""" + settings = parse_config(_config()) + self.assertEqual(settings.segment("EXP-END").date_keyword, "DATE-END") + with self.assertRaisesRegex(ValueError, "EXP-BEGIN, EXP-END"): + settings.segment("EXP-MID") + + def test_date_keyword_is_optional(self): + """Leave the date keyword out when a segment does not name one.""" + config = _config() + del config["segments"][0]["date_keyword"] + self.assertIsNone(parse_config(config).segments[0].date_keyword) + + +class UniquenessTests(unittest.TestCase): + """Duplicates are errors, never silently renamed or merged.""" + + def test_duplicate_within_a_segment(self): + """Reject the same FITS keyword twice in one segment.""" + with self.assertRaisesRegex(ConfigError, + "ADCPOS1 appears twice in segment"): + parse_config(_with_card(fits="ADCPOS1")) + + def test_duplicate_across_segments(self): + """Reject a keyword two segments share, since they merge into one header.""" + config = _config() + config["segments"][1]["cards"][0]["fits"] = "ADCPOS1" + with self.assertRaisesRegex(ConfigError, + "'EXP-BEGIN' and 'EXP-END'"): + parse_config(config) + + def test_date_keyword_counts_toward_uniqueness(self): + """Reject two segments that both claim DATE-BEG.""" + config = _config() + config["segments"][1]["date_keyword"] = "DATE-BEG" + with self.assertRaisesRegex(ConfigError, "DATE-BEG appears"): + parse_config(config) + + def test_duplicate_segment_name(self): + """Reject two segments with the same name.""" + config = _config() + config["segments"][1]["name"] = "EXP-BEGIN" + with self.assertRaisesRegex(ConfigError, "appears twice"): + parse_config(config) + + def test_mapping_form_is_refused(self): + """Refuse mappings, where YAML would hide a duplicate key.""" + text = """ +segments: + - name: S + cards: + ADCPOS1: {keyword: a.b.c, type: float} + ADCPOS1: {keyword: a.b.d, type: float} +""" + with self.assertRaisesRegex(ConfigError, "not a mapping"): + parse_config(yaml.safe_load(text)) + with self.assertRaisesRegex(ConfigError, "not a mapping"): + parse_config({"segments": {"S": {"cards": []}}}) + + +class KeywordValidationTests(unittest.TestCase): + """FITS keyword names follow the standard.""" + + def test_rejects_malformed_names(self): + """Reject lowercase, over-long and illegal-character names.""" + for name in ("adcpos", "TOOLONGNAME", "ADC.POS", "ADC POS"): + with self.subTest(name=name): + with self.assertRaisesRegex(ConfigError, "1-8 characters"): + parse_config(_with_card(fits=name)) + + def test_accepts_hyphen_and_underscore(self): + """Allow the full FITS character set.""" + parse_config(_with_card(fits="AD-C_9")) + + def test_rejects_reserved(self): + """Reject structural, commentary, WCS and time keywords.""" + for name in ("SIMPLE", "NAXIS2", "COMMENT", "HISTORY", "CONTINUE", + "END", "CTYPE1", "CRVAL2A", "PC1_2", "DATE-OBS", + "DATE-BEG", "OBJECT", "BZERO", "TFORM12", "OBSGEO-X"): + with self.subTest(name=name): + self.assertTrue(is_reserved(name)) + with self.assertRaisesRegex(ConfigError, "reserved"): + parse_config(_with_card(fits=name)) + + def test_near_misses_are_not_reserved(self): + """Do not over-match: these merely resemble reserved names.""" + for name in ("NAXISX", "EXPTIME", "CTYPE", "DATEX", "ADCPOS1"): + with self.subTest(name=name): + self.assertFalse(is_reserved(name)) + + def test_date_keyword_must_be_a_time_keyword(self): + """Let a segment name only one of the standard date keywords.""" + config = _config() + config["segments"][0]["date_keyword"] = "STARTTM" + with self.assertRaisesRegex(ConfigError, "date_keyword"): + parse_config(config) + + +class CardValidationTests(unittest.TestCase): + """Per-card field checks.""" + + def test_source_keyword_must_be_qualified(self): + """Require group, daemon and keyword.""" + for keyword in ("adc.positionvalue", "positionvalue", "hsfei.adc.%"): + with self.subTest(keyword=keyword): + with self.assertRaisesRegex(ConfigError, "."): + parse_config(_with_card(keyword=keyword)) + + def test_unknown_type_and_mode(self): + """Name the accepted choices.""" + with self.assertRaisesRegex(ConfigError, "float, int, bool, str"): + parse_config(_with_card(type="double")) + with self.assertRaisesRegex(ConfigError, "interpolate, nearest"): + parse_config(_with_card(mode="closest")) + + def test_no_interpolating_bool_or_str(self): + """Refuse to interpolate a value that has no in-between.""" + for kind in ("bool", "str"): + with self.subTest(kind=kind): + with self.assertRaisesRegex(ConfigError, "cannot interpolate"): + parse_config(_with_card(type=kind, mode="interpolate")) + + def test_format_is_checked_against_type(self): + """Reject a spec the type cannot take, or one that will not parse back.""" + with self.assertRaisesRegex(ConfigError, "not valid"): + parse_config(_with_card(type="int", format=".3s")) + with self.assertRaisesRegex(ConfigError, "read back"): + parse_config(_with_card(type="float", format=",.2f")) + with self.assertRaisesRegex(ConfigError, "bool"): + parse_config(_with_card(type="bool", format="d")) + + def test_comment_length(self): + """Reject a comment that would be truncated on the card.""" + parse_config(_with_card(comment="x" * MAX_COMMENT)) + with self.assertRaisesRegex(ConfigError, "at most"): + parse_config(_with_card(comment="x" * (MAX_COMMENT + 1))) + + def test_non_positive_window(self): + """Reject a window that could never contain a sample.""" + with self.assertRaisesRegex(ConfigError, "positive"): + parse_config(_with_card(window_s=0)) + + def test_empty_segments(self): + """Require at least one segment, and at least one card in each.""" + with self.assertRaisesRegex(ConfigError, "non-empty"): + parse_config(_config(segments=[])) + with self.assertRaisesRegex(ConfigError, "non-empty"): + parse_config(_config(segments=[{"name": "S", "cards": []}])) + + +class BuildSourceTests(unittest.TestCase): + """The source section is validated like the keygrabber's sink.""" + + SOURCE = {"type": "influxdb", "url": "http://influx:8086", "org": "o", + "bucket": "b", "token_env": "KEYHEADER_TEST_TOKEN"} + + def test_token_in_file_is_rejected(self): + """Keep the credential out of the config file.""" + with self.assertRaisesRegex(ConfigError, "source.token must not appear"): + build_source({**self.SOURCE, "token": "secret"}) + + def test_missing_token_env(self): + """Fail clearly when the named variable is unset.""" + with mock.patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(ConfigError, "source has no token"): + build_source(self.SOURCE) + + def test_unsupported_type(self): + """Name the one backend that exists.""" + with self.assertRaisesRegex(ConfigError, "unsupported source type"): + build_source({**self.SOURCE, "type": "sqlite"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_keyheader_daemon.py b/tests/test_keyheader_daemon.py new file mode 100644 index 0000000..d094014 --- /dev/null +++ b/tests/test_keyheader_daemon.py @@ -0,0 +1,262 @@ +"""Tests for the keyheader daemon, with a fake source in place of InfluxDB. + +The unit cases call the daemon directly. The end-to-end case starts it on ZMQ +and requests a segment over the wire, as a camera daemon would. +""" +from __future__ import annotations + +import os +import socket +import tempfile +import threading +import unittest +from datetime import datetime, timedelta, timezone +from typing import Dict, List, Optional, Tuple + +import yaml + +from libby import Client +from libby.config import ConfigError +from libby.keygrabber import Sample +from libby.keyheader import SERVICE_KEY, KeyheaderDaemon, parse_config, parse_time +from libby.keyheader.source import SourceReadError + +T0 = datetime(2026, 11, 11, 8, 15, 0, tzinfo=timezone.utc) + +SEGMENTS = [ + {"name": "EXP-BEGIN", "date_keyword": "DATE-BEG", + "cards": [ + {"fits": "ADCPOS", "keyword": "hsfei.adc.positionvalue", + "type": "float", "format": ".2f", "comment": "ADC position", + "mode": "interpolate"}, + {"fits": "ADCSTAT", "keyword": "hsfei.adc.status", "type": "str"}, + {"fits": "PRESSURE", "keyword": "hsfei.press.value", "type": "float", + "comment": "Dewar pressure"}, + ]}, +] + + +def _free_endpoint() -> str: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return f"tcp://127.0.0.1:{probe.getsockname()[1]}" + + +class _FakeSource: + """Serves canned samples per (peer, keyword), and can be told to fail.""" + + def __init__(self, samples: Dict[Tuple[str, str], List[Sample]]) -> None: + self.samples = samples + self.failing: set = set() + self.reads: List[Tuple[str, datetime, Optional[datetime]]] = [] + self._lock = threading.Lock() + + def connect(self) -> None: + """Nothing to open.""" + + def is_connected(self) -> bool: + """Always reachable.""" + return True + + def read(self, keyword, start, stop=None, group=None, peer=None): + """Return the canned samples inside the range.""" + with self._lock: + self.reads.append((keyword, start, stop)) + if keyword in self.failing: + raise SourceReadError("backend unreachable") + return [s for s in self.samples.get((peer, keyword), []) + if s.group == group and start <= s.timestamp + and (stop is None or s.timestamp < stop)] + + def close(self) -> None: + """Nothing to release.""" + + +def _sample(peer: str, keyword: str, seconds: float, value) -> Sample: + return Sample(keyword=keyword, group="hsfei", peer=peer, value=value, + units=None, timestamp=T0 + timedelta(seconds=seconds)) + + +def _source() -> _FakeSource: + return _FakeSource({ + ("adc", "positionvalue"): [_sample("adc", "positionvalue", -5, 1.0), + _sample("adc", "positionvalue", 5, 2.0)], + ("adc", "status"): [_sample("adc", "status", -3, "Ready")], + }) + + +class _TestKeyheader(KeyheaderDaemon): + """Keyheader reading a fake source instead of InfluxDB.""" + + source: _FakeSource + + def make_source(self): + """Return the injected source.""" + return self.source + + +class SegmentTests(unittest.TestCase): + """The daemon builds a segment's entries from the source.""" + + def setUp(self): + self.source = _source() + self.daemon = KeyheaderDaemon() + self.daemon.configure(parse_config({"segments": SEGMENTS}), self.source) + + def tearDown(self): + self.daemon.on_stop() + + def test_segment_entries(self): + """Interpolate, take nearest, and blank what is missing.""" + entries = self.daemon.segment("EXP-BEGIN", T0) + self.assertEqual([e["keyword"] for e in entries], + ["COMMENT", "DATE-BEG", "ADCPOS", "ADCSTAT", "PRESSURE"]) + by_name = {e["keyword"]: e for e in entries} + self.assertEqual(by_name["ADCPOS"]["value"], 1.5) + self.assertEqual(by_name["ADCSTAT"]["value"], "Ready") + self.assertIsNone(by_name["PRESSURE"]["value"]) + self.assertEqual(by_name["PRESSURE"]["comment"], + "no sample: Dewar pressure") + self.assertEqual(self.daemon.counters.blank_cards, 1) + + def test_read_failure_blanks_one_card(self): + """Blank only the card whose read failed, not the whole request.""" + self.source.failing.add("status") + by_name = {e["keyword"]: e for e in self.daemon.segment("EXP-BEGIN", T0)} + self.assertEqual(by_name["ADCSTAT"]["comment"], "read failed") + self.assertEqual(by_name["ADCPOS"]["value"], 1.5) + self.assertEqual(self.daemon.counters.read_errors, 1) + + def test_unknown_segment(self): + """Name the segments that do exist.""" + with self.assertRaisesRegex(ValueError, "EXP-BEGIN"): + self.daemon.segment("EXP-END", T0) + + def test_request_payload(self): + """Parse the request and reply with the entries and the moment.""" + # pylint: disable-next=protected-access + reply = self.daemon._serve({"segment": "EXP-BEGIN", + "time": "2026-11-11T08:15:00Z"}) + self.assertTrue(reply["ok"]) + self.assertEqual(reply["time"], "2026-11-11T08:15:00.000") + self.assertEqual(len(reply["cards"]), 5) + + def test_request_needs_segment_and_time(self): + """Reject a request missing either field.""" + # pylint: disable=protected-access + with self.assertRaisesRegex(ValueError, "segment"): + self.daemon._serve({"time": "2026-11-11T08:15:00Z"}) + with self.assertRaisesRegex(ValueError, "time"): + self.daemon._serve({"segment": "EXP-BEGIN"}) + + +class ParseTimeTests(unittest.TestCase): + """Request times are ISO 8601, defaulting to UTC.""" + + def test_forms(self): + """Accept Z, an offset, or no zone at all.""" + for text in ("2026-11-11T08:15:00Z", "2026-11-11T08:15:00+00:00", + "2026-11-11T08:15:00", "2026-11-11T00:15:00-08:00"): + with self.subTest(text=text): + self.assertEqual(parse_time(text), T0) + + def test_rejects_garbage(self): + """Fail clearly on a time that is not ISO 8601.""" + with self.assertRaisesRegex(ValueError, "ISO 8601"): + parse_time("yesterday") + + +class ReloadTests(unittest.TestCase): + """reload swaps segments, and refuses source changes.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with + self.path = os.path.join(self.tmp.name, "keyheader.yaml") + self._write(SEGMENTS) + self.daemon = _TestKeyheader.from_config_file(self.path) + self.daemon.configure(parse_config(self.daemon._config), _source()) # pylint: disable=protected-access + + def tearDown(self): + self.daemon.on_stop() + self.tmp.cleanup() + + def _write(self, segments, source=None) -> None: + config = {"peer_id": "keyheader", "group_id": "hispec", + "source": source or {"type": "influxdb"}, + "segments": segments} + with open(self.path, "w", encoding="utf-8") as handle: + yaml.safe_dump(config, handle) + + def test_reload_adds_a_segment(self): + """Pick up a new segment without a restart.""" + extra = {"name": "EXP-END", "cards": [ + {"fits": "ADCPOSE", "keyword": "hsfei.adc.positionvalue", + "type": "float", "mode": "prior"}]} + self._write(SEGMENTS + [extra]) + self.daemon._reload() # pylint: disable=protected-access + entries = self.daemon.segment("EXP-END", T0) + self.assertEqual(entries[-1]["value"], 1.0) + + def test_reload_refuses_source_change(self): + """Refuse a change that needs a new connection.""" + self._write(SEGMENTS, source={"type": "influxdb", "url": "elsewhere"}) + with self.assertRaisesRegex(ConfigError, "restart"): + self.daemon._reload() # pylint: disable=protected-access + + def test_bad_file_leaves_segments_alone(self): + """Keep serving the old segments when the new file is invalid.""" + self._write([{"name": "BAD", "cards": [ + {"fits": "SIMPLE", "keyword": "a.b.c", "type": "float"}]}]) + with self.assertRaises(ConfigError): + self.daemon._reload() # pylint: disable=protected-access + self.daemon.segment("EXP-BEGIN", T0) + + +class ZmqEndToEndTests(unittest.TestCase): + """A client requests a segment over the wire.""" + + def setUp(self): + self.endpoint = _free_endpoint() + self.daemon = _TestKeyheader.from_config({ + "peer_id": "keyheaderzmq", + "group_id": "hispec", + "transport": "zmq", + "bind": self.endpoint, + "address_book": {}, + "discovery_enabled": False, + "segments": SEGMENTS, + }) + self.daemon.source = _source() + self.daemon.start() + + def tearDown(self): + self.daemon.stop() + + def test_request_segment(self): + """Answer the service with entries, and serve control keywords.""" + peer = "hispec.keyheaderzmq" + with Client.zmq(bind=_free_endpoint(), + address_book={peer: self.endpoint}) as client: + # pylint: disable-next=protected-access + envelope = client._libby.rpc( + peer, SERVICE_KEY, + {"segment": "EXP-BEGIN", "time": "2026-11-11T08:15:00Z"}, + ttl_ms=5000) + reply = envelope.get("resp", envelope) + self.assertTrue(reply.get("ok"), reply) + self.assertEqual(reply["cards"][2], + {"keyword": "ADCPOS", "value": 1.5, + "comment": "ADC position"}) + self.assertEqual(client.get(f"{peer}.segments"), "EXP-BEGIN") + self.assertEqual(client.get(f"{peer}.requests"), 1) + + envelope = client._libby.rpc( # pylint: disable=protected-access + peer, SERVICE_KEY, {"segment": "NOPE", "time": "2026-11-11"}, + ttl_ms=5000) + reply = envelope.get("resp", envelope) + self.assertFalse(reply.get("ok")) + self.assertIn("unknown segment", reply["error"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_keyheader_values.py b/tests/test_keyheader_values.py new file mode 100644 index 0000000..fa9dba3 --- /dev/null +++ b/tests/test_keyheader_values.py @@ -0,0 +1,193 @@ +"""Unit tests for keyheader value acquisition and card entry building.""" +from __future__ import annotations + +import unittest +from datetime import datetime, timedelta, timezone +from typing import Any, List + +from libby.keygrabber import Sample +from libby.keyheader import (Acquired, Mode, ValueType, acquire, card_entry, + fits_date, query_range, segment_entries) +from libby.keyheader.config import CardConfig, SegmentConfig + +T0 = datetime(2026, 11, 11, 8, 15, 0, tzinfo=timezone.utc) + + +def _at(seconds: float, value: Any) -> Sample: + return Sample(keyword="positionvalue", group="hsfei", peer="adc", + value=value, units="mm", + timestamp=T0 + timedelta(seconds=seconds)) + + +def _card(kind: ValueType = ValueType.FLOAT, fmt=None, + comment: str = "ADC position") -> CardConfig: + return CardConfig(fits="ADCPOS", group="hsfei", peer="adc", + keyword="positionvalue", type=kind, format=fmt, + comment=comment, mode=Mode.NEAREST, window_s=60.0) + + +SAMPLES: List[Sample] = [_at(-10, 1.0), _at(-4, 2.0), _at(6, 4.0), _at(20, 9.0)] + + +class QueryRangeTests(unittest.TestCase): + """Each mode fetches only the side of the moment it can use.""" + + def test_prior_includes_the_moment(self): + """Run just past the moment, since a source's stop is exclusive.""" + start, stop = query_range(T0, Mode.PRIOR, 60) + self.assertEqual(start, T0 - timedelta(seconds=60)) + self.assertGreater(stop, T0) + self.assertLess(stop - T0, timedelta(milliseconds=1)) + + def test_subsequent_starts_at_the_moment(self): + """Start exactly at the moment.""" + self.assertEqual(query_range(T0, Mode.SUBSEQUENT, 60), + (T0, T0 + timedelta(seconds=60))) + + def test_nearest_and_interpolate_span_both_sides(self): + """Look a window either side.""" + for mode in (Mode.NEAREST, Mode.INTERPOLATE): + self.assertEqual(query_range(T0, mode, 60), + (T0 - timedelta(seconds=60), + T0 + timedelta(seconds=60))) + + +class AcquireTests(unittest.TestCase): + """Each mode picks the right sample, or says why it cannot.""" + + def test_prior(self): + """Take the latest sample at or before the moment.""" + self.assertEqual(acquire(SAMPLES, T0, Mode.PRIOR), Acquired(2.0)) + + def test_subsequent(self): + """Take the earliest sample at or after the moment.""" + self.assertEqual(acquire(SAMPLES, T0, Mode.SUBSEQUENT), Acquired(4.0)) + + def test_nearest(self): + """Take whichever side is closer in time.""" + self.assertEqual(acquire(SAMPLES, T0, Mode.NEAREST), Acquired(2.0)) + self.assertEqual(acquire(SAMPLES, T0 + timedelta(seconds=2), + Mode.NEAREST), Acquired(4.0)) + + def test_nearest_tie_prefers_earlier(self): + """Break a tie toward what was already true at the moment.""" + self.assertEqual(acquire(SAMPLES, T0 + timedelta(seconds=1), + Mode.NEAREST), Acquired(2.0)) + + def test_interpolate(self): + """Interpolate linearly between the neighbours.""" + result = acquire(SAMPLES, T0, Mode.INTERPOLATE) + self.assertAlmostEqual(result.value, 2.0 + 2.0 * 0.4) + + def test_exact_hit(self): + """Return a sample stamped exactly at the moment, in every mode.""" + samples = [_at(-5, 1.0), _at(0, 3.0), _at(5, 5.0)] + for mode in Mode: + with self.subTest(mode=mode): + self.assertEqual(acquire(samples, T0, mode), Acquired(3.0)) + + def test_interpolate_falls_back_to_one_side(self): + """Use the last value when no later sample is stored yet.""" + before_only = [s for s in SAMPLES if s.timestamp <= T0] + self.assertEqual(acquire(before_only, T0, Mode.INTERPOLATE), + Acquired(2.0)) + + def test_interpolate_non_numeric(self): + """Refuse to interpolate strings, rather than raising.""" + samples = [_at(-1, "a"), _at(1, "b")] + self.assertEqual(acquire(samples, T0, Mode.INTERPOLATE).reason, + "not numeric") + + def test_missing_reasons(self): + """Say which side was missing.""" + self.assertEqual(acquire([], T0, Mode.NEAREST).reason, "no sample") + self.assertEqual(acquire([_at(5, 1.0)], T0, Mode.PRIOR).reason, + "no prior sample") + self.assertEqual(acquire([_at(-5, 1.0)], T0, Mode.SUBSEQUENT).reason, + "no later sample") + + def test_nulls_are_ignored(self): + """Skip a null, which says nothing about the keyword's state.""" + samples = [_at(-5, 1.0), _at(-1, None)] + self.assertEqual(acquire(samples, T0, Mode.PRIOR), Acquired(1.0)) + + +class CardEntryTests(unittest.TestCase): + """Values become typed entries; problems become blank cards.""" + + def test_float_with_format(self): + """Apply the format's precision and keep a number.""" + entry = card_entry(_card(fmt=".3f"), Acquired(1.23456)) + self.assertEqual(entry, {"keyword": "ADCPOS", "value": 1.235, + "comment": "ADC position"}) + + def test_int_rounds_stored_float(self): + """Round, because InfluxDB stores every integer as a float.""" + entry = card_entry(_card(ValueType.INT), Acquired(41.9999)) + self.assertEqual(entry["value"], 42) + self.assertIsInstance(entry["value"], int) + + def test_bool(self): + """Accept a bool, or a 0/1 a backend stored as a number.""" + self.assertIs(card_entry(_card(ValueType.BOOL), Acquired(True))["value"], + True) + self.assertIs(card_entry(_card(ValueType.BOOL), Acquired(0.0))["value"], + False) + + def test_str_with_format(self): + """Apply a string format, e.g. to truncate.""" + entry = card_entry(_card(ValueType.STR, fmt=".5"), Acquired("Ready now")) + self.assertEqual(entry["value"], "Ready") + + def test_missing_value_is_blank_with_reason(self): + """Keep the card, with no value and a comment saying why.""" + entry = card_entry(_card(), Acquired.missing("no sample")) + self.assertEqual(entry, {"keyword": "ADCPOS", "value": None, + "comment": "no sample: ADC position"}) + + def test_wrong_type_is_blank(self): + """Blank a value that cannot be the configured type.""" + self.assertEqual(card_entry(_card(), Acquired("text"))["comment"], + "not a float: ADC position") + self.assertEqual( + card_entry(_card(ValueType.BOOL), Acquired(2))["comment"], + "not a bool: ADC position") + + def test_non_finite_is_blank(self): + """Blank NaN and infinity, which FITS cannot represent.""" + entry = card_entry(_card(), Acquired(float("nan"))) + self.assertIsNone(entry["value"]) + self.assertTrue(entry["comment"].startswith("not finite")) + + def test_blank_comment_is_truncated_to_fit(self): + """Keep the combined reason and comment within one card.""" + entry = card_entry(_card(comment="x" * 47), Acquired.missing("no sample")) + self.assertEqual(len(entry["comment"]), 47) + + +class SegmentEntryTests(unittest.TestCase): + """A segment starts with its COMMENT and date keyword.""" + + def test_layout(self): + """COMMENT, then the date keyword, then cards in config order.""" + segment = SegmentConfig(name="EXP-BEGIN", date_keyword="DATE-BEG", + date_comment="start", cards=(_card(),)) + moment = T0 + timedelta(microseconds=123456) + entries = segment_entries(segment, moment, [Acquired(1.5)]) + self.assertEqual(entries[0], {"keyword": "COMMENT", + "value": "EXP-BEGIN 2026-11-11T08:15:00.123", + "comment": ""}) + self.assertEqual(entries[1], {"keyword": "DATE-BEG", + "value": "2026-11-11T08:15:00.123", + "comment": "start"}) + self.assertEqual(entries[2]["keyword"], "ADCPOS") + + def test_fits_date_converts_to_utc(self): + """Write the date in UTC whatever zone the moment arrived in.""" + pacific = timezone(timedelta(hours=-8)) + self.assertEqual(fits_date(datetime(2026, 11, 11, 0, 0, tzinfo=pacific)), + "2026-11-11T08:00:00.000") + + +if __name__ == "__main__": + unittest.main() From dd2cfff7403d37c449f237ea5c9e194b184952e9 Mon Sep 17 00:00:00 2001 From: Don Neill Date: Wed, 11 Nov 2026 16:47:43 -0800 Subject: [PATCH 6/9] Add an example keyheader config A commented keyheader.example.yaml with EXP-BEGIN and EXP-END segments mapped from real hispec keywords. It ships as package data, and a test parses it so a change to the validation rules cannot leave it stale. Co-Authored-By: Claude Opus 5.5 --- docs/source/keyheader.md | 3 +- libby/keyheader/keyheader.example.yaml | 92 ++++++++++++++++++++++++++ pyproject.toml | 1 + tests/test_keyheader_config.py | 19 ++++++ 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 libby/keyheader/keyheader.example.yaml diff --git a/docs/source/keyheader.md b/docs/source/keyheader.md index c1679ad..a1e374b 100644 --- a/docs/source/keyheader.md +++ b/docs/source/keyheader.md @@ -26,7 +26,8 @@ Like the keygrabber, it is an ordinary `LibbyDaemon`: `SIGTERM` or the ## Config A keyheader config is an ordinary daemon config plus `source`, `defaults` and -`segments`. +`segments`. A fuller, commented example ships with the package as +`libby/keyheader/keyheader.example.yaml`. ```yaml peer_id: keyheader diff --git a/libby/keyheader/keyheader.example.yaml b/libby/keyheader/keyheader.example.yaml new file mode 100644 index 0000000..dd4260d --- /dev/null +++ b/libby/keyheader/keyheader.example.yaml @@ -0,0 +1,92 @@ +# Example keyheader config. +# +# keyheader -c keyheader.yaml +# +# An ordinary LibbyDaemon config (peer_id, group_id, transport, ...) plus the +# three keyheader sections: source, defaults and segments. See +# docs/source/keyheader.md for every field and the validation rules. + +peer_id: keyheader +group_id: hispec +transport: rabbitmq +rabbitmq_url: amqp://localhost + +# The InfluxDB the keygrabber writes to. The token is never written here: +# token_env names the environment variable that holds it, and a `token` key +# is rejected. A read-only token is enough. +source: + type: influxdb + url: http://localhost:8086 + org: hispec + bucket: HISPEC_keyhistory + token_env: HISPEC_INFLUX_TOKEN + timeout_ms: 10000 + +# Threads reading a segment's cards in parallel +workers: 4 + +# Applied to every card unless its segment or the card itself overrides them. +# window_s must exceed the keygrabber's interval for a keyword (including its +# failure backoff), or a quiet keyword comes back blank. +defaults: + mode: nearest + window_s: 300 + +# A list, not a mapping: YAML silently keeps only the last of two identical +# keys, which would hide a duplicate. Segment names are what a requesting +# daemon asks for, and FITS keywords must be unique across ALL segments, +# because the requesting daemon merges them into one header. +segments: + - name: EXP-BEGIN + date_keyword: DATE-BEG + date_comment: UTC start of exposure + cards: + - fits: YJDETTMP + keyword: hsyj.yjdettherm.tA_detector + type: float + format: ".3f" + comment: "YJ detector temperature [K]" + mode: interpolate + - fits: YJPENTMP + keyword: hsyj.yjdettherm.tB_penguin + type: float + format: ".3f" + comment: "YJ penguin temperature [K]" + mode: interpolate + - fits: YJTHMCON + keyword: hsyj.yjdettherm.isconnected + type: bool + comment: "YJ detector temp controller connected" + - fits: YJTHMSTA + keyword: hsyj.yjdettherm.1_detector_status + type: str + format: ".40" # truncate, so it stays on one card + comment: "YJ detector heater loop status" + - fits: SRSPOWER + keyword: hspower.yjcal2.power_7_SRS + type: float + format: ".1f" + comment: "SRS outlet power draw [W]" + - fits: SRSON + keyword: hspower.yjcal2.state_7_SRS + type: bool + comment: "SRS outlet switched on" + + # At the end of an exposure the next sample is usually not stored yet, so + # take the last value at or before the moment + - name: EXP-END + date_keyword: DATE-END + date_comment: UTC end of exposure + defaults: + mode: prior + cards: + - fits: YJDETTME + keyword: hsyj.yjdettherm.tA_detector + type: float + format: ".3f" + comment: "YJ detector temperature at end [K]" + - fits: YJPENTME + keyword: hsyj.yjdettherm.tB_penguin + type: float + format: ".3f" + comment: "YJ penguin temperature at end [K]" \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5535c41..74e46a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ exclude = ["peers*", "package*"] [tool.setuptools.package-data] "libby.cli" = ["*.yaml"] +"libby.keyheader" = ["*.yaml"] "libby.telemetry_backend" = ["templates/*"] [project.scripts] diff --git a/tests/test_keyheader_config.py b/tests/test_keyheader_config.py index 1515c28..4f69e69 100644 --- a/tests/test_keyheader_config.py +++ b/tests/test_keyheader_config.py @@ -3,6 +3,7 @@ import copy import os +from pathlib import Path import unittest from typing import Any, Dict from unittest import mock @@ -247,6 +248,24 @@ def test_empty_segments(self): parse_config(_config(segments=[{"name": "S", "cards": []}])) +class ExampleConfigTests(unittest.TestCase): + """The shipped example stays valid as the rules change.""" + + def test_example_parses(self): + """Parse keyheader.example.yaml and check its source section.""" + path = Path(__file__).parent.parent / "libby" / "keyheader" / \ + "keyheader.example.yaml" + with path.open(encoding="utf-8") as handle: + settings = parse_config(yaml.safe_load(handle)) + self.assertEqual([s.name for s in settings.segments], + ["EXP-BEGIN", "EXP-END"]) + with mock.patch.dict(os.environ, {"HISPEC_INFLUX_TOKEN": "t"}): + try: + build_source(settings.source) + except ImportError: + self.skipTest("influxdb extra not installed") + + class BuildSourceTests(unittest.TestCase): """The source section is validated like the keygrabber's sink.""" From 9c910a15678f98340fccbe18387e4c50b0e6f4f7 Mon Sep 17 00:00:00 2001 From: Don Neill Date: Thu, 12 Nov 2026 10:21:19 -0800 Subject: [PATCH 7/9] convert yaml to flow style so each keyword is on one line in keyheader.example.yaml --- libby/keyheader/keyheader.example.yaml | 53 +++++++------------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/libby/keyheader/keyheader.example.yaml b/libby/keyheader/keyheader.example.yaml index dd4260d..faa7ef8 100644 --- a/libby/keyheader/keyheader.example.yaml +++ b/libby/keyheader/keyheader.example.yaml @@ -36,41 +36,22 @@ defaults: # keys, which would hide a duplicate. Segment names are what a requesting # daemon asks for, and FITS keywords must be unique across ALL segments, # because the requesting daemon merges them into one header. +# +# Each card is written on one line in YAML flow style. Inside { } the +# characters [ ] { } , and ": " have meaning, so quote any value that holds +# them; comments are always quoted here for that reason. segments: - name: EXP-BEGIN date_keyword: DATE-BEG date_comment: UTC start of exposure cards: - - fits: YJDETTMP - keyword: hsyj.yjdettherm.tA_detector - type: float - format: ".3f" - comment: "YJ detector temperature [K]" - mode: interpolate - - fits: YJPENTMP - keyword: hsyj.yjdettherm.tB_penguin - type: float - format: ".3f" - comment: "YJ penguin temperature [K]" - mode: interpolate - - fits: YJTHMCON - keyword: hsyj.yjdettherm.isconnected - type: bool - comment: "YJ detector temp controller connected" - - fits: YJTHMSTA - keyword: hsyj.yjdettherm.1_detector_status - type: str - format: ".40" # truncate, so it stays on one card - comment: "YJ detector heater loop status" - - fits: SRSPOWER - keyword: hspower.yjcal2.power_7_SRS - type: float - format: ".1f" - comment: "SRS outlet power draw [W]" - - fits: SRSON - keyword: hspower.yjcal2.state_7_SRS - type: bool - comment: "SRS outlet switched on" + - {fits: YJDETTMP, keyword: hsyj.yjdettherm.tA_detector, type: float, format: ".3f", comment: "YJ detector temperature [K]", mode: interpolate} + - {fits: YJPENTMP, keyword: hsyj.yjdettherm.tB_penguin, type: float, format: ".3f", comment: "YJ penguin temperature [K]", mode: interpolate} + - {fits: YJTHMCON, keyword: hsyj.yjdettherm.isconnected, type: bool, comment: "YJ detector temp controller connected"} + # format ".40" truncates the string, so it stays on one card + - {fits: YJTHMSTA, keyword: hsyj.yjdettherm.1_detector_status, type: str, format: ".40", comment: "YJ detector heater loop status"} + - {fits: SRSPOWER, keyword: hspower.yjcal2.power_7_SRS, type: float, format: ".1f", comment: "SRS outlet power draw [W]"} + - {fits: SRSON, keyword: hspower.yjcal2.state_7_SRS, type: bool, comment: "SRS outlet switched on"} # At the end of an exposure the next sample is usually not stored yet, so # take the last value at or before the moment @@ -80,13 +61,5 @@ segments: defaults: mode: prior cards: - - fits: YJDETTME - keyword: hsyj.yjdettherm.tA_detector - type: float - format: ".3f" - comment: "YJ detector temperature at end [K]" - - fits: YJPENTME - keyword: hsyj.yjdettherm.tB_penguin - type: float - format: ".3f" - comment: "YJ penguin temperature at end [K]" \ No newline at end of file + - {fits: YJDETTME, keyword: hsyj.yjdettherm.tA_detector, type: float, format: ".3f", comment: "YJ detector temperature at end [K]"} + - {fits: YJPENTME, keyword: hsyj.yjdettherm.tB_penguin, type: float, format: ".3f", comment: "YJ penguin temperature at end [K]"} From bc0b341df399accbd5260f3cdcd2dd00d6e1d646 Mon Sep 17 00:00:00 2001 From: Don Neill Date: Thu, 12 Nov 2026 13:33:36 -0800 Subject: [PATCH 8/9] Change uniqueness rule so that it applies only to a given segment. This way keywords can be used for other cameras. --- docs/source/keyheader.md | 14 ++++---- libby/keyheader/config.py | 24 ++++++------- libby/keyheader/daemon.py | 48 +++++++++++++++++--------- libby/keyheader/keyheader.example.yaml | 6 ++-- libby/keyheader/main.py | 2 +- tests/test_keyheader_config.py | 17 +++++---- 6 files changed, 63 insertions(+), 48 deletions(-) diff --git a/docs/source/keyheader.md b/docs/source/keyheader.md index a1e374b..6f578f8 100644 --- a/docs/source/keyheader.md +++ b/docs/source/keyheader.md @@ -121,10 +121,11 @@ Config load rejects: (`SIMPLE`, `NAXISn`, `END`, ...), commentary (`COMMENT`, `HISTORY`, `CONTINUE`), the general descriptive ones (`OBJECT`, `TELESCOP`, `DATE`, ...), and the WCS and time keywords. Keyheader writes COMMENT itself. -- a FITS keyword used twice, either within one segment or across segments. - The requesting daemon merges segments into one header, and FITS forbids a - repeated keyword, so the error names both segments rather than renaming - either. +- a FITS keyword used twice within one segment. Different segments may use + the same keywords, so several camera daemons can each have their own + segments with the same names. A daemon that adds two segments to one header + must pick segments that do not overlap, since FITS forbids a repeated + keyword. - a repeated segment name - `interpolate` on a `bool` or `str` card - a `format` the type cannot take, or a numeric format whose output does not @@ -198,8 +199,9 @@ Use `end=True`, or build a fresh `fits.Header(cards)`. Plain cards, which moves the segment's COMMENT below the cards it introduces. The requesting daemon is still responsible for checking the final header -against the FITS standard. For example, it should not add a second segment -that names the same keywords. +against the FITS standard. Keyheader only checks keywords within a segment, +so a daemon that adds more than one segment to a header must make sure they +use different keywords, or replace a card rather than appending a duplicate. ### Values and formats diff --git a/libby/keyheader/config.py b/libby/keyheader/config.py index b7e16da..bd2f010 100644 --- a/libby/keyheader/config.py +++ b/libby/keyheader/config.py @@ -14,7 +14,7 @@ import re from dataclasses import dataclass from enum import Enum -from typing import Any, Dict, Mapping, Optional, Sequence, Tuple +from typing import Any, Dict, Mapping, Optional, Sequence, Set, Tuple from ..config import ConfigError # The keygrabber's validators, so both daemons phrase config errors alike @@ -271,11 +271,12 @@ def _parse_card(label: str, raw: Any, defaults: Mapping[str, Any]) -> CardConfig def _check_unique(segments: Sequence[SegmentConfig]) -> None: - """Reject a repeated segment name, or a FITS keyword used twice anywhere. + """Reject a repeated segment name, or a FITS keyword used twice in a segment. - Keywords are unique across every segment, not just within one, because the - requesting daemon merges segments into one header and FITS forbids a - repeated keyword. + Keywords need only be unique within a segment. Several camera daemons can + each have their own segment using the same keywords, so two segments may + share a name; a daemon that adds two segments to one header must choose + segments that do not overlap. """ seen_segments: Dict[str, int] = {} for index, segment in enumerate(segments): @@ -286,19 +287,16 @@ def _check_unique(segments: Sequence[SegmentConfig]) -> None: f"segments[{index}])") seen_segments[segment.name] = index - owners: Dict[str, str] = {} for segment in segments: + seen: Set[str] = set() names = [card.fits for card in segment.cards] if segment.date_keyword is not None: names.insert(0, segment.date_keyword) for name in names: - if name in owners: - where = ("twice in segment " + repr(segment.name) - if owners[name] == segment.name - else f"in segments {owners[name]!r} and " - f"{segment.name!r}") - raise ConfigError(f"FITS keyword {name} appears {where}") - owners[name] = segment.name + if name in seen: + raise ConfigError(f"FITS keyword {name} appears twice in " + f"segment {segment.name!r}") + seen.add(name) def _parse_source_name(label: str, raw: Any) -> Tuple[str, str, str]: diff --git a/libby/keyheader/daemon.py b/libby/keyheader/daemon.py index f43741a..7c5d324 100644 --- a/libby/keyheader/daemon.py +++ b/libby/keyheader/daemon.py @@ -1,10 +1,11 @@ -"""The keyheader daemon: answer a named segment at a moment with FITS cards.""" +"""The keyheader daemon: answer a named segment request at a moment with FITS cards.""" from __future__ import annotations import threading from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, List, Mapping, Optional +from typing import Any, List, Mapping, Optional, cast from ..config import ConfigError, DaemonConfigLoader, with_env_overrides from ..daemon import LibbyDaemon @@ -17,6 +18,15 @@ SERVICE_KEY = "keyheader.segment" +@dataclass(frozen=True) +class ConfigFile: + """Where a daemon's config came from, so ``reload`` can re-read it.""" + + path: str + daemon_id: Optional[str] = None + env_prefix: Optional[str] = None + + class Counters: # pylint: disable=too-few-public-methods """Tallies the daemon reports, guarded for cross-thread increments.""" @@ -54,9 +64,8 @@ def __init__(self) -> None: self._source: Optional[Source] = None self._pool: Optional[ThreadPoolExecutor] = None self._source_healthy = True - self._config_path: Optional[str] = None - self._daemon_id: Optional[str] = None - self._env_prefix: Optional[str] = None + # Set by from_config_file; None for a daemon built from a mapping + self.config_file: Optional[ConfigFile] = None def on_start(self) -> None: """Parse config, open the source, and start answering requests.""" @@ -83,18 +92,22 @@ def configure(self, settings: KeyheaderConfig, exercise :meth:`segment` without a transport or a database. """ self._settings = settings - self._source = source if source is not None else self.make_source() - self._source.connect() + if source is None: + source = self.make_source() + # Held before connecting, so on_stop still closes it if connect fails + self._source = source + source.connect() self._pool = ThreadPoolExecutor(max_workers=settings.workers, thread_name_prefix="keyheader-read") @classmethod def from_config_file(cls, path, daemon_id=None, *, env_prefix=None): """Build from a file, remembering where it came from for ``reload``.""" - daemon = super().from_config_file(path, daemon_id, env_prefix=env_prefix) - daemon._config_path = str(path) - daemon._daemon_id = daemon_id - daemon._env_prefix = env_prefix + # The base returns an instance of cls, but type checkers resolve the + # super() call to LibbyDaemon, which lacks these attributes + daemon = cast("KeyheaderDaemon", super().from_config_file( + path, daemon_id, env_prefix=env_prefix)) + daemon.config_file = ConfigFile(str(path), daemon_id, env_prefix) return daemon def make_source(self) -> Source: @@ -193,12 +206,13 @@ def _reload(self) -> None: A change to ``source`` or ``workers`` does, and is refused rather than half-applied. A request in flight keeps the settings it started with. """ - if self._config_path is None: + origin = self.config_file + if origin is None: raise LibbyError("this keyheader was not built from a config file") - config = DaemonConfigLoader(self._config_path).get_daemon_config( - self._daemon_id) - if self._env_prefix: - config = with_env_overrides(config, prefix=self._env_prefix) + config = DaemonConfigLoader(origin.path).get_daemon_config( + origin.daemon_id) + if origin.env_prefix: + config = with_env_overrides(config, prefix=origin.env_prefix) settings = parse_config(config) running = self._require() if (dict(settings.source) != dict(running.source) @@ -207,7 +221,7 @@ def _reload(self) -> None: "restart instead") self._settings = settings self.logger.info("reloaded %d segments from %s", - len(settings.segments), self._config_path) + len(settings.segments), origin.path) def _require(self) -> KeyheaderConfig: if self._settings is None: diff --git a/libby/keyheader/keyheader.example.yaml b/libby/keyheader/keyheader.example.yaml index faa7ef8..7eb30fd 100644 --- a/libby/keyheader/keyheader.example.yaml +++ b/libby/keyheader/keyheader.example.yaml @@ -34,8 +34,10 @@ defaults: # A list, not a mapping: YAML silently keeps only the last of two identical # keys, which would hide a duplicate. Segment names are what a requesting -# daemon asks for, and FITS keywords must be unique across ALL segments, -# because the requesting daemon merges them into one header. +# daemon asks for. FITS keywords must be unique within a segment, but +# different segments may reuse them, e.g. one set of segments per camera. +# EXP-BEGIN and EXP-END below use distinct names because one camera adds +# both to the same header. # # Each card is written on one line in YAML flow style. Inside { } the # characters [ ] { } , and ": " have meaning, so quote any value that holds diff --git a/libby/keyheader/main.py b/libby/keyheader/main.py index 58bdb8e..aa882b4 100644 --- a/libby/keyheader/main.py +++ b/libby/keyheader/main.py @@ -19,7 +19,7 @@ def main(argv: Optional[List[str]] = None) -> int: """Run keyheader until it is stopped. Return an exit code.""" parser = argparse.ArgumentParser( prog="keyheader", - description="Answer FITS header segments from stored keyword values", + description="Answer FITS header segment requests from stored keyword values", ) parser.add_argument("-c", "--config", required=True, help="path to the keyheader config (YAML or JSON)") diff --git a/tests/test_keyheader_config.py b/tests/test_keyheader_config.py index 4f69e69..d984860 100644 --- a/tests/test_keyheader_config.py +++ b/tests/test_keyheader_config.py @@ -121,20 +121,19 @@ def test_duplicate_within_a_segment(self): "ADCPOS1 appears twice in segment"): parse_config(_with_card(fits="ADCPOS1")) - def test_duplicate_across_segments(self): - """Reject a keyword two segments share, since they merge into one header.""" + def test_same_keyword_in_different_segments(self): + """Allow segments to share keywords, e.g. one segment per camera.""" config = _config() config["segments"][1]["cards"][0]["fits"] = "ADCPOS1" - with self.assertRaisesRegex(ConfigError, - "'EXP-BEGIN' and 'EXP-END'"): - parse_config(config) + settings = parse_config(config) + self.assertEqual(settings.segments[1].cards[0].fits, "ADCPOS1") - def test_date_keyword_counts_toward_uniqueness(self): - """Reject two segments that both claim DATE-BEG.""" + def test_same_date_keyword_in_different_segments(self): + """Allow two cameras' segments to both write DATE-BEG.""" config = _config() config["segments"][1]["date_keyword"] = "DATE-BEG" - with self.assertRaisesRegex(ConfigError, "DATE-BEG appears"): - parse_config(config) + self.assertEqual(parse_config(config).segments[1].date_keyword, + "DATE-BEG") def test_duplicate_segment_name(self): """Reject two segments with the same name.""" From 8c9db61c1fb125e30ea03b3f2e63b74624930337 Mon Sep 17 00:00:00 2001 From: Don Neill Date: Thu, 12 Nov 2026 14:49:17 -0800 Subject: [PATCH 9/9] Update keyheader.md to include one-line examples --- docs/source/keyheader.md | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/docs/source/keyheader.md b/docs/source/keyheader.md index 6f578f8..3663d84 100644 --- a/docs/source/keyheader.md +++ b/docs/source/keyheader.md @@ -52,29 +52,23 @@ segments: - name: EXP-BEGIN date_keyword: DATE-BEG cards: - - fits: ADCPOS1 - keyword: hsfei.adc.positionvalue1 - type: float - format: ".3f" - comment: "ADC prism 1 position [deg]" - mode: interpolate - - fits: ADCMOVE - keyword: hsfei.adc.ismoving - type: bool - comment: "ADC in motion" + - {fits: ADCPOS1, keyword: hsfei.adc.positionvalue1, type: float, format: ".3f", comment: "ADC prism 1 position [deg]", mode: interpolate} + - {fits: ADCMOVE, keyword: hsfei.adc.ismoving, type: bool, comment: "ADC in motion"} - name: EXP-END date_keyword: DATE-END defaults: mode: prior cards: - - fits: ADCPOS1E - keyword: hsfei.adc.positionvalue1 - type: float - format: ".3f" - comment: "ADC prism 1 position at end [deg]" + - {fits: ADCPOS1E, keyword: hsfei.adc.positionvalue1, type: float, format: ".3f", comment: "ADC prism 1 position at end [deg]"} ``` +Each card is one line in YAML flow style, `{key: value, ...}`. Block style +works too, and the two can be mixed. Inside `{ }` the characters `[ ] { } ,` +and `: ` have meaning, so quote any value that contains them. An unquoted +comment such as `ADC position [deg]` is a parse error, which is why the +comments above are always quoted. + ### source Validated exactly like the keygrabber's `sink`: `type: influxdb`, and the token