From f97c0f2498930ac1d5d100f9a90a2290ebb93bc9 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:24:43 +0100 Subject: [PATCH 1/9] feat(profile): read a bundle through flux-stats' exact-YAML reader bundle.py kept its own port of the engine's exact loader and canonical form. It now reads each file with flux_stats.exact_yaml.load_documents, under one ReadBudget for the bundle, and canonicalises the block with flux_stats.canonical, both through the backend seam. The file listing, the per-file byte cap and the BundleError events stay. The [profile] extra and the install message name flux-stats[yaml]>=0.7.0,<0.8. --- fluid_build/cli/profile_cmd.py | 2 +- fluid_build/profiler/_stats.py | 58 ++++- fluid_build/profiler/bundle.py | 307 +++-------------------- pyproject.toml | 14 +- tests/profiler/_fake_backend.py | 18 +- tests/profiler/test_bundle.py | 27 +- tests/profiler/test_bundle_flux_stats.py | 14 +- tests/profiler/test_cli_profile.py | 2 +- tests/profiler/test_flux_stats_pin.py | 12 +- tests/profiler/test_runner.py | 2 +- 10 files changed, 147 insertions(+), 309 deletions(-) diff --git a/fluid_build/cli/profile_cmd.py b/fluid_build/cli/profile_cmd.py index 69758b28..44a6e19f 100644 --- a/fluid_build/cli/profile_cmd.py +++ b/fluid_build/cli/profile_cmd.py @@ -216,7 +216,7 @@ def run(args: argparse.Namespace, logger: logging.Logger) -> int: try: expose_id = runner.resolve_expose_id(contract, args.expose_id) out = _out_path(args.out, expose_id, args.source) - block = bundle.read_bundle_block(bundle_dir, expose_id) if bundle_dir else None + block = bundle.read_bundle_block(bundle_dir, expose_id, backend) if bundle_dir else None result = runner.profile_expose( contract, expose_id, diff --git a/fluid_build/profiler/_stats.py b/fluid_build/profiler/_stats.py index d6475b95..8c11bead 100644 --- a/fluid_build/profiler/_stats.py +++ b/fluid_build/profiler/_stats.py @@ -52,6 +52,7 @@ __all__ = [ "BlockRefused", + "BundleTextRefused", "DeclaredBlock", "DeclaredStatistic", "DocumentRefused", @@ -99,6 +100,18 @@ def __init__(self, code: str, message: str) -> None: self.code = code +class BundleTextRefused(ValueError): + """The registry's exact-YAML reader or canonical form refused a bundle's text. + + The message names the file and, where the registry gives one, the line and + column, with the registry's ``FLX-nnnn`` code. It never quotes a value. + """ + + def __init__(self, code: str, message: str) -> None: + super().__init__(f"{message} ({code})") + self.code = code + + class MeasureRefused(ValueError): """The registry refused a record while forming a declared statistic. @@ -303,6 +316,24 @@ def measure( def quantile_ranks_ok(self, k: int, level: Fraction, below: int, at_or_below: int) -> bool: """Whether a witness with these ranks, over *k* records, is the *level* quantile.""" + def read_budget(self) -> Any: + """A fresh budget for reading one bundle's documents, to share across its files.""" + + def read_documents(self, text: str, *, name: str, budget: Any) -> Tuple[Mapping[str, Any], ...]: + """Every mapping document in *text*, read as the FLUX engine reads a bundle. + + A float is the decimal of its source text and a timestamp stays a string. The + read is bounded by *budget* (aliases, merge keys, digits, depth). Raises + :class:`BundleTextRefused` for text it refuses, naming *name*. + """ + + def canonicalise(self, value: Any, *, name: str) -> Any: + """*value* in the engine's canonical form (KERNEL-SPEC 2.1). + + Every key is an NFC string, every string NFC, a decimal its string spelling. + Raises :class:`BundleTextRefused`, naming *name*, for a value with no canonical form. + """ + #: The most digits a number may need written out without an exponent: the #: engine's declared-number rule (flux-engine ``concordance/predicate.py`` @@ -387,7 +418,7 @@ class _FluxStatsBackend: def __init__(self) -> None: # The package first: when it is missing, this is the import that fails, # even with one of its submodules still in sys.modules. - from flux_stats import Parameters, definitions, metrics + from flux_stats import Parameters, canonical, definitions, exact_yaml, metrics from flux_stats.errors import FluxError from flux_stats.fixedpoint import Grid from flux_stats.profile import load_profile_text @@ -401,6 +432,8 @@ def __init__(self) -> None: self._category = category self._read_exact = read_exact self._value_domain = ValueDomain + self._exact_yaml = exact_yaml + self._canonical = canonical self._columns: Dict[Tuple[str, str], _Column] = {} self._grammar: Any = None @@ -439,6 +472,25 @@ def read_value(self, domain: str, value: Any, *, column: str, row_index: int) -> # A timestamp reads as a whole number of milliseconds since the epoch. return int(exact) if domain == "timestamp" else exact + def read_budget(self) -> Any: + return self._exact_yaml.ReadBudget() + + def read_documents(self, text: str, *, name: str, budget: Any) -> Tuple[Mapping[str, Any], ...]: + try: + documents: Tuple[Mapping[str, Any], ...] = self._exact_yaml.load_documents( + text, name=name, budget=budget + ) + except self._refused as error: + raise BundleTextRefused(error.code.code, error.detail or error.code.title) from None + return documents + + def canonicalise(self, value: Any, *, name: str) -> Any: + try: + return self._canonical.canonicalise(value) + except self._refused as error: + detail = error.detail or error.code.title + raise BundleTextRefused(error.code.code, f"{name}: {detail}") from None + def check_document(self, text: str) -> None: try: self._load_profile_text(text) @@ -652,7 +704,9 @@ def load_backend() -> StatisticsBackend: #: The flux-stats range ``[profile]`` names in ``pyproject.toml``. The CLI cannot read #: that file once installed, so the install message repeats it; #: ``tests/profiler/test_flux_stats_pin.py`` keeps the two equal. -FLUX_STATS_REQUIREMENT = "flux-stats>=0.6.2,<0.7" +#: TODO: the founder names the real version when flux-stats releases it; 0.7.0 is the +#: first release that holds ``edge_counts``, ``exact_yaml`` and ``quantile/2``. +FLUX_STATS_REQUIREMENT = "flux-stats[yaml]>=0.7.0,<0.8" #: The extra carries the range, and pip replaces a copy outside it (the 0.0.1 #: placeholder, or a release newer than this module knows). FLUX_STATS_INSTALL_COMMAND = "pip install 'data-product-forge[profile]'" diff --git a/fluid_build/profiler/bundle.py b/fluid_build/profiler/bundle.py index a72b12c3..7ed71c41 100644 --- a/fluid_build/profiler/bundle.py +++ b/fluid_build/profiler/bundle.py @@ -18,33 +18,28 @@ (:meth:`fluid_build.profiler._stats.StatisticsBackend.read_block`), so the profiler implements no grammar of its own and imports no registry here. -The file is read the way the FLUX engine reads it, and the block is returned in -the engine's canonical form, so the registry's parser sees what the engine's -sees. This is a port, not an import (the profiler never imports the engine): - -* the loader is ``flux_engine/seam/resolve.py`` ``ExactLoader``: a float scalar - is its source text as a ``Decimal``, a timestamp stays a string, and an - integer too long to form is refused; -* the canonical form is ``flux_engine/core/canonical.py`` ``coerce_key``, - ``canonicalise`` and ``_render_decimal`` (KERNEL-SPEC 2.1): every mapping key - is an NFC string (``True`` is ``"true"``, ``None`` is ``"null"``, an integer - its digits), every string is NFC, a ``Decimal`` becomes its string spelling, - and a float key or an integer outside the JCS safe range is refused. The - engine parses the block from that form (``stages/pipeline.py``). +The text is read the way the FLUX engine reads it, by flux-stats' exact-YAML +reader (``flux_stats.exact_yaml``) and canonical form (``flux_stats.canonical``), +both reached through the backend seam: a float scalar is its source text as a +decimal, a timestamp stays a string, an integer too long to form is refused, and +the block is returned in the engine's canonical form (KERNEL-SPEC 2.1), so the +registry's parser sees what the engine's sees. The reader bounds what a document +may write out (aliases, merge keys, digits, depth), with one budget for every +document of the bundle. A refusal names the file and the keys it looked for, never a value from a record. Nothing in a bundle is a record, but an operator's terminal output -travels, so a YAML error is reduced to its line number. +travels, so a YAML error is reduced to what the registry's reader reports. """ from __future__ import annotations -import sys import unicodedata from dataclasses import dataclass -from decimal import Decimal, InvalidOperation from pathlib import Path -from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Dict, List, Mapping, Optional, Tuple + +from fluid_build.profiler._stats import BundleTextRefused, StatisticsBackend __all__ = [ "BUNDLE_NAMESPACE", @@ -64,33 +59,6 @@ _MAX_BYTES = 4 * 1024 * 1024 -#: The largest integer JCS (RFC 8785) writes exactly: 2**53 - 1. -_JCS_MAX_SAFE_INTEGER = 2**53 - 1 - -#: The most digits an integer may have: CPython's own limit on int/text conversion. -_MOST_INTEGER_DIGITS: int = sys.int_info.default_max_str_digits - -_INTEGER_TAG = "tag:yaml.org,2002:int" -_FLOAT_TAG = "tag:yaml.org,2002:float" -_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" -_MERGE_TAG = "tag:yaml.org,2002:merge" - -#: The most digits a number in a block may need written out: the engine's rule for a -#: declared number, flux-stats' ``MOST_DIGITS``, counted as its -#: ``reading.digits_written_out`` counts (:func:`_digits_written_out`). It is checked -#: before a decimal is written out, since writing ``1E+300000000`` out takes gigabytes. -_MOST_NUMBER_DIGITS = 1000 - -#: The most values, and the deepest nesting, a block may hold once written out. YAML -#: reads an alias as a shared reference, so a kilobyte of aliases can stand for -#: billions of values; a real block holds a few hundred, a few levels deep. -_MOST_BLOCK_VALUES = 100_000 -_DEEPEST_BLOCK = 64 -#: The most characters of a key a refusal's path names: the path is built for every -#: value, so an aliased long key must not be copied whole at each level. -_KEY_IN_PATH = 40 - - class BundleError(Exception): """A bundle that cannot give a fidelity block. ``event`` is a stable code.""" @@ -126,228 +94,25 @@ def _documents(directory: Path) -> List[Path]: ] -class _Merges: - """The pairs YAML merge keys may still copy while one bundle is read.""" - - def __init__(self) -> None: - self.left = _MOST_BLOCK_VALUES - - -def _exact_loader(name: str, merges: _Merges) -> Any: - """The engine's ``ExactLoader``, built on first use so PyYAML stays off the cold path. - - Its merge keys draw on *merges*, which every document of one bundle shares. - """ - import yaml - - def merged_parts(value_node: Any) -> List[Any]: - """The mappings a merge key's value names: one mapping, or a list of them.""" - if isinstance(value_node, yaml.MappingNode): - return [value_node] - if isinstance(value_node, yaml.SequenceNode): - return [part for part in value_node.value if isinstance(part, yaml.MappingNode)] - return [] - - class ExactLoader(yaml.SafeLoader): - """A float is its source text as a Decimal, a timestamp a string, a long int refused.""" - - def flatten_mapping(self, node: Any) -> None: - # A merge key (``<<``) copies its mappings' pairs while the document is - # built, before _canonicalise counts a value, and a chain of merges doubles - # at each level. So every pair a merge copies, in any document of the - # bundle, is counted against one bound before the copy is made. - for key_node, value_node in node.value: - if key_node.tag != _MERGE_TAG: - continue - for part in merged_parts(value_node): - self.flatten_mapping(part) - merges.left -= len(part.value) - if merges.left < 0: - raise refuse( - node, - f"merge keys (<<) copy more than {_MOST_BLOCK_VALUES} pairs " - "in the bundle", - ) - super().flatten_mapping(node) - - def refuse(node: Any, why: str) -> BundleError: - mark = node.start_mark - return BundleError( - "profile_bundle_unreadable", - f"{name} line {mark.line + 1}, column {mark.column + 1}: {why}", - {"next": "write the value as the FLUX engine reads it, or name another bundle"}, - ) - - def construct_float(loader: Any, node: Any) -> Decimal: - text = str(loader.construct_scalar(node)).replace("_", "") - if ":" in text: - raise refuse(node, "a sexagesimal number has no decimal spelling; write plain decimal") - try: - value = Decimal(text) - except InvalidOperation: - raise refuse(node, "a scalar read as a float is not a decimal number") from None - if not value.is_finite(): - raise refuse(node, "an infinite or not-a-number value is not allowed") - return value - - def construct_int(loader: Any, node: Any) -> int: - too_long = f"an integer of more than {_MOST_INTEGER_DIGITS} digits is too long to read" - digits = sum(ch.isdigit() for ch in str(loader.construct_scalar(node))) - if digits > _MOST_INTEGER_DIGITS: - raise refuse(node, too_long) - try: - value = int(loader.construct_yaml_int(node)) - except (ValueError, IndexError): - raise refuse(node, "a scalar read as an integer has no integer value") from None - if abs(value) >= 10**_MOST_INTEGER_DIGITS: - raise refuse(node, too_long) - return value - - ExactLoader.add_constructor(_FLOAT_TAG, construct_float) - ExactLoader.add_constructor(_INTEGER_TAG, construct_int) - - def construct_timestamp(loader: Any, node: Any) -> Any: - return loader.construct_scalar(node) - - ExactLoader.add_constructor(_TIMESTAMP_TAG, construct_timestamp) - return ExactLoader - - def _nfc(text: str) -> str: return unicodedata.normalize("NFC", text) -def _digits_written_out(value: Decimal) -> int: - """The digits *value* needs written out, as flux-stats' ``digits_written_out`` counts.""" - digits = len(value.as_tuple().digits) - exponent = value.adjusted() - digits + 1 - if not value: - return 1 # zero, however its exponent is spelled - if exponent >= 0: - return digits + exponent - return max(digits, -exponent) - - -def _render_decimal(value: Decimal, name: str, where: str) -> str: - """The declared scale is kept; negative zero is folded onto the positive spelling.""" - if _digits_written_out(value) > _MOST_NUMBER_DIGITS: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} holds a number of more than {_MOST_NUMBER_DIGITS} digits " - "written out, which FLUX does not read", - {"next": "write a shorter number"}, - ) - rendered = format(value, "f") - if rendered.startswith("-") and value == 0: - rendered = rendered[1:] - return rendered - - -def _coerce_key(key: Any, name: str, where: str) -> str: - if isinstance(key, str): - return _nfc(key) - if isinstance(key, bool): - return "true" if key else "false" - if isinstance(key, float): - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} has a float key, which has no canonical spelling", - {"next": "quote the key so it is a string"}, - ) - if isinstance(key, int): - return str(key) - if key is None: - return "null" - if isinstance(key, Decimal): - return _render_decimal(key, name, where) - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} has a key of class {type(key).__name__!r}", - {"next": "quote the key so it is a string"}, - ) - - -class _Budget: - """The values left to write one block out with.""" - - def __init__(self) -> None: - self.left = _MOST_BLOCK_VALUES - - def spend(self, name: str, where: str, depth: int) -> None: - """Count one value at *depth*; refuse a block past either bound.""" - self.left -= 1 - if self.left < 0: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} holds more than {_MOST_BLOCK_VALUES} values once its " - "YAML aliases are written out", - {"next": "write the block without aliases, or name another bundle"}, - ) - if depth > _DEEPEST_BLOCK: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} nests deeper than {_DEEPEST_BLOCK} levels", - {"next": "write the block without aliases, or name another bundle"}, - ) - - -def _canonicalise( - value: Any, name: str, where: str, budget: Optional[_Budget] = None, depth: int = 0 -) -> Any: - """*value* as JSON-ready Python (KERNEL-SPEC 2.1), mirroring the engine's ``canonicalise``. - - Each value written out costs one of *budget*, so aliases cannot make the walk - write out more than :data:`_MOST_BLOCK_VALUES` values or go deeper than - :data:`_DEEPEST_BLOCK` levels. - """ - budget = budget if budget is not None else _Budget() - budget.spend(name, where, depth) - if isinstance(value, bool): - return value - if isinstance(value, int): - if abs(value) > _JCS_MAX_SAFE_INTEGER: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} holds an integer outside the range 2^53 - 1 that FLUX reads", - {"next": "write a smaller integer, or quote it as a string"}, - ) - return value - if isinstance(value, Decimal): - return _render_decimal(value, name, where) - if isinstance(value, str): - return _nfc(value) - if value is None: - return None - if isinstance(value, Mapping): - canonical: Dict[str, Any] = {} - for raw_key, raw_value in value.items(): - key = _coerce_key(raw_key, name, where) - if key in canonical: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} has two keys that both read as {key!r}", - {"next": "keep one of them"}, - ) - step = key if len(key) <= _KEY_IN_PATH else f"{key[:_KEY_IN_PATH]}..." - canonical[key] = _canonicalise(raw_value, name, f"{where}.{step}", budget, depth + 1) - return {key: canonical[key] for key in sorted(canonical)} - if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): - return [_canonicalise(item, name, where, budget, depth + 1) for item in value] - raise BundleError( +def _unreadable(message: str) -> BundleError: + return BundleError( "profile_bundle_unreadable", - f"{name}: {where} holds a value of class {type(value).__name__!r}", - {"next": "write plain YAML scalars, lists and mappings"}, + message, + {"next": "fix the document, or name another bundle directory"}, ) -def _load(path: Path, merges: _Merges) -> Tuple[Mapping[str, Any], ...]: - """Every mapping document in *path*, as the engine's ``load_documents_exact`` reads it. +def _load(path: Path, backend: StatisticsBackend, budget: Any) -> Tuple[Mapping[str, Any], ...]: + """Every mapping document in *path*, as flux-stats' ``load_documents`` reads it. A file may hold more than one ``---`` document; a :class:`BundleError` names the - file and the line of a document that cannot be read. + file, and the line where the registry's reader gives one, of a document that cannot + be read. """ - import yaml - try: if path.stat().st_size > _MAX_BYTES: raise BundleError( @@ -362,27 +127,12 @@ def _load(path: Path, merges: _Merges) -> Tuple[Mapping[str, Any], ...]: "profile_bundle_unreadable", f"{path.name} cannot be read: {reason}" ) from None try: - documents = list( - yaml.load_all(text, Loader=_exact_loader(path.name, merges)) # nosec B506 - SafeLoader - ) - except RecursionError: - raise BundleError( - "profile_bundle_unreadable", - f"{path.name} nests too deeply to read", - {"next": "fix the document, or name another bundle directory"}, - ) from None - except yaml.YAMLError as exc: - mark = getattr(exc, "problem_mark", None) - where = f" at line {mark.line + 1}" if mark is not None else "" - raise BundleError( - "profile_bundle_unreadable", - f"{path.name} is not valid YAML{where}", - {"next": "fix the document, or name another bundle directory"}, - ) from None - return tuple(document for document in documents if isinstance(document, Mapping)) + return backend.read_documents(text, name=path.name, budget=budget) + except BundleTextRefused as exc: + raise _unreadable(str(exc)) from None -def read_bundle_block(directory: Path, expose_id: str) -> BundleBlock: +def read_bundle_block(directory: Path, expose_id: str, backend: StatisticsBackend) -> BundleBlock: """The fidelity block the bundle's one Simulation declares for *expose_id*. Refuses, each with its own event, a directory with no Simulation or more than @@ -390,9 +140,9 @@ def read_bundle_block(directory: Path, expose_id: str) -> BundleBlock: not a mapping. """ simulations: List[Tuple[Path, Mapping[str, Any]]] = [] - merges = _Merges() # one bound for every document of the bundle + budget = backend.read_budget() # one bound for every document of the bundle for path in _documents(directory): - for document in _load(path, merges): + for document in _load(path, backend, budget): if document.get("kind") == "Simulation": simulations.append((path, document)) if not simulations: @@ -436,5 +186,8 @@ def read_bundle_block(directory: Path, expose_id: str) -> BundleBlock: f"{path.name}: {where} is not a mapping", {"next": "declare the fidelity block as FLUX's fidelity grammar spells it"}, ) - block = _canonicalise(block, path.name, where) + try: + block = backend.canonicalise(block, name=f"{path.name}: {where}") + except BundleTextRefused as exc: + raise _unreadable(str(exc)) from None return BundleBlock(raw=block, simulation=path.name, expose_id=expose_id) diff --git a/pyproject.toml b/pyproject.toml index 0bd79d18..b65b1450 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -404,16 +404,20 @@ lineage = [ # does. It is standard-library only and supports Python 3.10, so it needs no # marker. # -# flux-stats comes from PyPI. 0.6.2 is the first release that holds both the -# record reader the profiler uses and the fidelity-block grammar --bundle reads -# (flux_stats.fidelity); PyPI's 0.0.1 is a placeholder with no code. It is -# versioned 0.x, where a minor release may change the names +# flux-stats comes from PyPI. The profiler needs a release that holds the record +# reader, the fidelity-block grammar (flux_stats.fidelity), quantile/2, the output +# check for edge counts (flux_stats.edge_counts) and the exact-YAML reader and +# canonical form --bundle reads with (flux_stats.exact_yaml, flux_stats.canonical; +# the ``yaml`` extra brings PyYAML). PyPI's 0.0.1 is a placeholder with no code. It +# is versioned 0.x, where a minor release may change the names # fluid_build/profiler/_stats.py imports, so the range stops at the next minor. # That module repeats the requirement in its install message, and # tests/profiler/test_flux_stats_pin.py keeps the two equal. +# TODO: the founder names the real flux-stats version when it is released; 0.7.0 is +# the first release that holds these modules. profile = [ "duckdb>=1.5.0", - "flux-stats>=0.6.2,<0.7", + "flux-stats[yaml]>=0.7.0,<0.8", ] # AI + MCP together: restores the pre-0.28.0 base behaviour (the diff --git a/tests/profiler/_fake_backend.py b/tests/profiler/_fake_backend.py index 9358bcb5..9f73e8ff 100644 --- a/tests/profiler/_fake_backend.py +++ b/tests/profiler/_fake_backend.py @@ -29,7 +29,7 @@ from pathlib import Path from typing import Any, Callable, Dict, FrozenSet, List, Sequence, Tuple -from fluid_build.profiler._stats import DocumentRefused, ReadRefused +from fluid_build.profiler._stats import BundleTextRefused, DocumentRefused, ReadRefused FIXTURES = Path(__file__).parent / "fixtures" SCHEMA_PATH = FIXTURES / "profile-1.schema.json" @@ -212,3 +212,19 @@ def check_document(self, text: str) -> None: ) if errors: raise DocumentRefused("; ".join(error.message for error in errors[:3])) + + def read_budget(self) -> object: + return object() + + def read_documents(self, text: str, *, name: str, budget: object) -> Tuple[Dict[str, Any], ...]: + """Plain YAML: enough for a bundle's file listing, which the real reader tests cover.""" + import yaml + + try: + documents = list(yaml.safe_load_all(text)) + except yaml.YAMLError: + raise BundleTextRefused("FLX-3105", f"{name}: unreadable YAML") from None + return tuple(document for document in documents if isinstance(document, dict)) + + def canonicalise(self, value: Any, *, name: str) -> Any: + return value diff --git a/tests/profiler/test_bundle.py b/tests/profiler/test_bundle.py index 09499768..8bd7c7d9 100644 --- a/tests/profiler/test_bundle.py +++ b/tests/profiler/test_bundle.py @@ -7,8 +7,10 @@ import pytest -from fluid_build.profiler import bundle -from fluid_build.profiler.bundle import BundleError, read_bundle_block +from fluid_build.profiler import _stats, bundle +from fluid_build.profiler.bundle import BundleError + +flux_stats = pytest.importorskip("flux_stats") pytestmark = pytest.mark.unit @@ -28,6 +30,11 @@ """ +def read_bundle_block(directory, expose_id): + """The bundle's block, read through flux-stats' exact-YAML reader as the CLI reads it.""" + return bundle.read_bundle_block(directory, expose_id, _stats.load_backend()) + + def _write(directory, name, text): (directory / name).write_text(text, encoding="utf-8") @@ -108,7 +115,7 @@ def test_an_unreadable_document_names_the_file_and_the_line_and_quotes_nothing(t with pytest.raises(BundleError) as refused: read_bundle_block(tmp_path, "orders") assert refused.value.event == "profile_bundle_unreadable" - assert "simulation.flux.yml" in str(refused.value) and "line" in str(refused.value) + assert "simulation.flux.yml" in str(refused.value) and "unreadable YAML" in str(refused.value) assert "value-77" not in str(refused.value) @@ -192,10 +199,10 @@ def test_a_bare_float_key_is_its_decimal_spelling_as_the_engine_reads_it(tmp_pat def test_a_binary_float_key_is_refused_and_names_the_file(): # The loader never builds a float, so this is the canonical form's own guard. - with pytest.raises(BundleError) as refused: - bundle._canonicalise({0.5: "x"}, "s.flux.yml", "fidelity") - assert refused.value.event == "profile_bundle_unreadable" - assert "s.flux.yml" in str(refused.value) and "float key" in str(refused.value) + with pytest.raises(_stats.BundleTextRefused) as refused: + _stats.load_backend().canonicalise({0.5: "x"}, name="s.flux.yml: fidelity") + assert "s.flux.yml" in str(refused.value) and "float" in str(refused.value) + assert refused.value.code == "FLX-1001" def test_an_integer_outside_the_safe_range_is_refused(tmp_path): @@ -221,21 +228,21 @@ def test_a_block_whose_aliases_write_out_too_many_values_is_refused(tmp_path): with pytest.raises(BundleError) as refused: read_bundle_block(tmp_path, "orders") assert refused.value.event == "profile_bundle_unreadable" - assert "more than 100000 values" in str(refused.value) + assert "more than 1000000 values" in str(refused.value) def test_a_block_that_holds_itself_through_an_alias_is_refused(tmp_path): _write(tmp_path, "s.flux.yml", _simulation_with("r: &r [*r]")) with pytest.raises(BundleError) as refused: read_bundle_block(tmp_path, "orders") - assert "nests deeper than 64 levels" in str(refused.value) + assert "holds itself through an alias" in str(refused.value) def test_a_number_too_long_to_write_out_is_refused(tmp_path): _write(tmp_path, "s.flux.yml", _simulation_with("columns: {amount: {max: 1.0e+300000000}}")) with pytest.raises(BundleError) as refused: read_bundle_block(tmp_path, "orders") - assert "more than 1000 digits" in str(refused.value) + assert "more than 1000 digits" in str(refused.value) and "line" in str(refused.value) def test_a_document_nested_too_deeply_to_read_is_refused(tmp_path): diff --git a/tests/profiler/test_bundle_flux_stats.py b/tests/profiler/test_bundle_flux_stats.py index f90a52eb..6209920a 100644 --- a/tests/profiler/test_bundle_flux_stats.py +++ b/tests/profiler/test_bundle_flux_stats.py @@ -147,7 +147,7 @@ def _profile( ): chosen = properties if properties is not None else PROPERTIES directory = _bundle(tmp_path, chosen, bits, market_null) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) return profile_expose( CONTRACT, EXPOSE, @@ -628,7 +628,7 @@ def test_a_value_outside_the_declared_range_is_refused(tmp_path): def test_a_block_the_grammar_refuses_is_refused_before_any_record_is_read(tmp_path): properties = [{"property": "mean", "column": "nope", "tolerance": _ABSOLUTE}] directory = _bundle(tmp_path, properties) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) with pytest.raises(ProfileError) as refused: profile_expose( CONTRACT, @@ -645,7 +645,7 @@ def test_a_declaration_over_the_wrong_domain_is_refused(tmp_path): contract = json.loads(json.dumps(CONTRACT)) contract["exposes"][0]["contract"]["schema"][1]["type"] = "integer" # market as a number directory = _bundle(tmp_path, [PROPERTIES[4]]) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) with pytest.raises(ProfileError) as refused: profile_expose( contract, @@ -662,7 +662,7 @@ def test_a_declared_column_the_contract_types_unreadably_is_refused(tmp_path): contract = json.loads(json.dumps(CONTRACT)) contract["exposes"][0]["contract"]["schema"][0]["type"] = "double" directory = _bundle(tmp_path, [PROPERTIES[0]]) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) with pytest.raises(ProfileError) as refused: profile_expose( contract, @@ -677,7 +677,7 @@ def test_a_declared_column_the_contract_types_unreadably_is_refused(tmp_path): def test_the_seam_deduplicates_identical_properties(tmp_path): directory = _bundle(tmp_path, [PROPERTIES[0], PROPERTIES[0], PROPERTIES[6]]) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) declared = _stats.load_backend().read_block(block.raw, CONTRACT, EXPOSE) assert [s.description for s in declared.statistics] == ["mean/1 of 'amount'"] assert declared.columns == ("amount",) @@ -777,7 +777,7 @@ def test_a_flux_stats_without_the_grammar_refuses_before_reading(tmp_path, monke assert rc == 1 assert "profile_statistics_unavailable" in out assert "reads the fidelity block with flux-stats, and the installed flux-stats" in out - assert "lacks flux_stats.fidelity. It needs flux-stats>=0.6.2,<0.7" in out + assert "lacks flux_stats.fidelity. It needs flux-stats[yaml]>=0.7.0,<0.8" in out assert "next: pip install 'data-product-forge[profile]'" in out @@ -790,7 +790,7 @@ def test_a_block_written_with_bare_null_keys_and_bare_floats_profiles_as_the_quo text = path.read_text(encoding="utf-8") assert "'null':" in text and "level: '0.5'" in text path.write_text(text.replace("'null':", "null:").replace("level: '0.5'", "level: 0.5")) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) assert block.raw["columns"]["market"]["null"] == "fail" bare = profile_expose( CONTRACT, diff --git a/tests/profiler/test_cli_profile.py b/tests/profiler/test_cli_profile.py index 9fe8d4c2..9a88d2c4 100644 --- a/tests/profiler/test_cli_profile.py +++ b/tests/profiler/test_cli_profile.py @@ -102,7 +102,7 @@ def test_without_the_registry_nothing_is_read_and_nothing_written(workspace, cap assert rc == 1 assert "profile_statistics_unavailable" in out assert "and flux-stats is not installed" in out - assert "It needs flux-stats>=0.6.2,<0.7" in out + assert "It needs flux-stats[yaml]>=0.7.0,<0.8" in out assert "next: pip install 'data-product-forge[profile]'" in out assert "git+https" not in out assert not (workspace / "orders.profile.json").exists() diff --git a/tests/profiler/test_flux_stats_pin.py b/tests/profiler/test_flux_stats_pin.py index 73c6170e..1e5a492d 100644 --- a/tests/profiler/test_flux_stats_pin.py +++ b/tests/profiler/test_flux_stats_pin.py @@ -24,6 +24,10 @@ def _profile_flux_stats() -> list: return [r for r in requirements if g.requirement_name(r) == "flux-stats"] +def test_the_requirement_names_the_yaml_extra_for_the_bundle_reader(): + assert Requirement(_stats.FLUX_STATS_REQUIREMENT).extras == {"yaml"} + + def test_the_profile_extra_names_flux_stats_once(): assert len(_profile_flux_stats()) == 1 @@ -36,11 +40,11 @@ def test_the_requirement_is_a_range_of_released_versions(): requirement = Requirement(_stats.FLUX_STATS_REQUIREMENT) assert requirement.url is None # PyPI refuses a URL in Requires-Dist assert requirement.marker is None # flux-stats supports Python 3.10, the floor here - assert requirement.specifier.contains("0.6.2") - assert requirement.specifier.contains("0.6.9") + assert requirement.specifier.contains("0.7.0") + assert requirement.specifier.contains("0.7.9") assert not requirement.specifier.contains("0.0.1") # the placeholder holds no code - assert not requirement.specifier.contains("0.6.1") # no flux_stats.fidelity for --bundle - assert not requirement.specifier.contains("0.7.0") # a 0.x minor may rename what is imported + assert not requirement.specifier.contains("0.6.2") # no edge_counts, exact_yaml or quantile/2 + assert not requirement.specifier.contains("0.8.0") # a 0.x minor may rename what is imported def test_the_install_command_installs_the_extra_that_holds_flux_stats(): diff --git a/tests/profiler/test_runner.py b/tests/profiler/test_runner.py index c203f9f6..69d776f5 100644 --- a/tests/profiler/test_runner.py +++ b/tests/profiler/test_runner.py @@ -296,7 +296,7 @@ def version(name): with pytest.raises(StatisticsUnavailable) as caught: load_backend() message = str(caught.value) - assert f"and flux-stats {state}. It needs flux-stats>=0.6.2,<0.7;" in message + assert f"and flux-stats {state}. It needs flux-stats[yaml]>=0.7.0,<0.8;" in message assert "install it with: pip install 'data-product-forge[profile]'" in message From 21e1dd94f7d37703a573a3af7abf8cf13fb869c6 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:31:19 +0100 Subject: [PATCH 2/9] feat(profile): name format 1-1-0 when a profile holds a quantile/2 A profile names format 1-0-0 unless one of its statistics is quantile/2, which only format 1-1-0 carries, so every existing profile keeps its bytes. document.py learns the 1-1-0 URL and picks it from the tables it is given; a test keeps it equal to flux_stats.profile's. --- fluid_build/profiler/document.py | 32 ++++++++++++++++++++++++++++---- tests/profiler/test_document.py | 23 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/fluid_build/profiler/document.py b/fluid_build/profiler/document.py index eff96844..f4c309a3 100644 --- a/fluid_build/profiler/document.py +++ b/fluid_build/profiler/document.py @@ -1,7 +1,7 @@ # Copyright 2025 Agentics Transformation Limited # SPDX-License-Identifier: Apache-2.0 -"""The profile document: PROFILE-SPEC format 1-0-0, its canonical bytes and digest. +"""The profile document: PROFILE-SPEC formats 1-0-0 and 1-1-0, its canonical bytes and digest. flux-engine ``docs/spec/PROFILE-SPEC.md`` §1 fixes the shape: the OpenLineage facet envelope (``_producer``, ``_schemaURL``), the grid (``encoding``), the @@ -15,6 +15,10 @@ digest from one implementation. The file holds exactly those bytes, so its sha256 is the digest a Simulation pins. +A profile names format 1-0-0, unless it holds a ``quantile/2`` statistic, which only +format 1-1-0 carries (:func:`schema_url_for`): a profile without one keeps the bytes it +had. + A document that breaks a rule is refused here, before anything is written. The refusal names where in the document the problem is, never a value. """ @@ -33,16 +37,26 @@ "MAX_FRACTIONAL_BITS", "PRODUCER_REPOSITORY", "SCHEMA_URL", + "SCHEMA_URL_1_1_0", "build_document", "canonical_bytes", "check_fractional_bits", "digest_of", "producer_uri", + "schema_url_for", ] -#: The format this module writes. A later format is a new URL (PROFILE-SPEC §8). +#: The format a profile names unless it needs the next one. A later format is a new URL +#: (PROFILE-SPEC §8). SCHEMA_URL = "https://agenticstiger.github.io/flux/engine/spec/profile/1-0-0/FluxProfile.json" +#: Format 1-1-0 adds the ``quantile/2`` statistic and changes nothing 1-0-0 says +#: (``flux_stats.profile.SCHEMA_URL_1_1_0``; a test keeps the two equal). +SCHEMA_URL_1_1_0 = "https://agenticstiger.github.io/flux/engine/spec/profile/1-1-0/FluxProfile.json" + +#: The statistics only format 1-1-0 holds. +_ADDED_IN_1_1_0 = frozenset({"quantile/2"}) + #: ``_producer`` names this repository, at the release that wrote the profile. PRODUCER_REPOSITORY = "https://github.com/Agenticstiger/forge-cli" @@ -73,6 +87,16 @@ def producer_uri(version: str) -> str: return f"{PRODUCER_REPOSITORY}/tree/v{version}/fluid_build/profiler" +def schema_url_for(tables: Mapping[str, Mapping[str, Any]]) -> str: + """The format *tables* need: 1-1-0 when any statistic is ``quantile/2``, else 1-0-0.""" + for table in tables.values(): + for column in table.get("columns", {}).values(): + for statistic in column.get("statistics", ()): + if statistic.get("definition") in _ADDED_IN_1_1_0: + return SCHEMA_URL_1_1_0 + return SCHEMA_URL + + def build_document( *, producer: str, @@ -80,13 +104,13 @@ def build_document( tables: Mapping[str, Mapping[str, Any]], fractional_bits: int = DEFAULT_FRACTIONAL_BITS, ) -> Dict[str, Any]: - """Assemble a format 1-0-0 document from tables already checked by *rule*.""" + """Assemble a document from tables already checked by *rule*, in the format they need.""" check_fractional_bits(fractional_bits) if not tables: raise DocumentError("tables holds at least one table") return { "_producer": producer, - "_schemaURL": SCHEMA_URL, + "_schemaURL": schema_url_for(tables), "encoding": {"fractionalBits": fractional_bits, "rounding": "floor"}, "disclosureRules": {rule.name: rule.to_document()}, "tables": {expose_id: dict(table) for expose_id, table in tables.items()}, diff --git a/tests/profiler/test_document.py b/tests/profiler/test_document.py index cc4e2552..a2864e1b 100644 --- a/tests/profiler/test_document.py +++ b/tests/profiler/test_document.py @@ -15,6 +15,7 @@ from fluid_build.profiler.disclosure import ACRO_DEFAULT from fluid_build.profiler.document import ( SCHEMA_URL, + SCHEMA_URL_1_1_0, DocumentError, build_document, canonical_bytes, @@ -79,6 +80,28 @@ def test_a_built_document_is_format_1_0_0(): assert document["disclosureRules"] == {"acro-default": ACRO_DEFAULT.to_document()} +def _with_statistic(definition): + table = _table() + table["columns"]["amount"]["statistics"] = [{"definition": definition, "groups": []}] + return table + + +def test_a_quantile_with_bins_needs_format_1_1_0_and_nothing_else_does(): + assert _document(tables={"orders": _with_statistic("quantile/2")})["_schemaURL"] == ( + SCHEMA_URL_1_1_0 + ) + for definition in ("mean/1", "marginalHistogram/1", "levelHistogram/1", "quantile/1"): + document = _document(tables={"orders": _with_statistic(definition)}) + assert document["_schemaURL"] == SCHEMA_URL + assert _document()["_schemaURL"] == SCHEMA_URL + + +def test_the_formats_are_the_ones_flux_stats_reads(): + profile = pytest.importorskip("flux_stats.profile") + assert SCHEMA_URL == profile.SCHEMA_URL + assert SCHEMA_URL_1_1_0 == profile.SCHEMA_URL_1_1_0 + + def test_the_producer_names_this_repository_at_the_release(): assert producer_uri("0.31.0") == ( "https://github.com/Agenticstiger/forge-cli/tree/v0.31.0/fluid_build/profiler" From 7efad59a18712991f163b20b8cef7c058e2b0670 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:31:19 +0100 Subject: [PATCH 3/9] feat(profile): check a column's edge counts with flux-stats, and write quantile/2 A quantile that declares bins is quantile/2: its group states the bin that holds the order statistic and the records below and through it, with no witness. The backend seam gains check_edge_counts, and the runner passes every histogram, level histogram and quantile of a column to it in one call, with the rows a null groupBy key leaves out as one more group that is counted and never written. Retires the profiler's own copies of that rule: the sibling and crowded checks in the runner, the histogram and quantile branches of _check_group, publish_cells, check_quantile, written_status and the backend's quantile_ranks_ok. Mean and support keep their own checks and secondary suppression. --- fluid_build/profiler/_stats.py | 194 ++++++++++++-- fluid_build/profiler/disclosure.py | 197 +++----------- fluid_build/profiler/runner.py | 305 +++++++++++++--------- tests/profiler/test_bundle_flux_stats.py | 269 +++++++++++++++---- tests/profiler/test_disclosure.py | 146 +---------- tests/profiler/test_flux_stats_backend.py | 17 ++ 6 files changed, 629 insertions(+), 499 deletions(-) diff --git a/fluid_build/profiler/_stats.py b/fluid_build/profiler/_stats.py index 8c11bead..1b6c65ed 100644 --- a/fluid_build/profiler/_stats.py +++ b/fluid_build/profiler/_stats.py @@ -56,6 +56,8 @@ "DeclaredBlock", "DeclaredStatistic", "DocumentRefused", + "EdgeDecision", + "EdgeStatistic", "MeasureRefused", "MeasuredGroup", "MeasuredStatistic", @@ -64,6 +66,7 @@ "StatisticDefinition", "StatisticsBackend", "StatisticsUnavailable", + "WITHHELD_ROUNDING", "load_backend", ] @@ -132,8 +135,10 @@ class DeclaredStatistic: ``shape`` holds the document keys that say which statistic it is beyond its definition and column: ``{"bins": [...]}`` (the declared edges, as decimal strings), ``{"levels": [...]}`` (the declared levels in cell order), - ``{"level": "0.5"}`` or nothing. ``columns`` is every column it reads, the - ``groupBy`` column included, and ``read_as`` the domain each is read in. + ``{"level": "0.5"}`` (and ``bins`` too, for a ``quantile/2``) or nothing. + ``columns`` is every column it reads, the ``groupBy`` column included, and + ``read_as`` the domain each is read in. ``edges`` are the declared bins + placed on the grid, for a histogram and a ``quantile/2``. """ identifier: str @@ -143,6 +148,7 @@ class DeclaredStatistic: shape: Mapping[str, Any] columns: Tuple[str, ...] read_as: Mapping[str, str] + edges: Tuple[int, ...] = () #: The registry's parsed declaration; only the registry's own seam reads it. _declaration: Any = field(default=None, repr=False, compare=False) @@ -180,13 +186,16 @@ class MeasuredGroup: statistic's quantities), with every count exact and no ``status``. The other fields hold what the disclosure rules read: ``cells`` (a histogram's bin or level counts), ``member_holders`` (the records each member of a support is - held by), ``quantile`` (witness, records strictly below it, records at or - below it) and, for a mean, the group's exact values and their grid points. + held by), ``quantile`` (a ``quantile/1``'s witness, records strictly below + it, records at or below it) and, for a mean, the group's exact values and + their grid points. ``statistic`` is the registry's measured group, which the + check for counts at edges (:meth:`StatisticsBackend.check_edge_counts`) reads. """ key: str document: Dict[str, Any] k: int + statistic: Any = field(default=None, repr=False, compare=False) cells: Tuple[int, ...] = () member_holders: Tuple[int, ...] = () quantile: Optional[Tuple[Optional[int], int, int]] = None @@ -211,6 +220,56 @@ class MeasuredStatistic: left_out_unread: bool = False +#: Why a count is withheld for its rounding, in the registry's own word +#: (``flux_stats.edge_counts.ROUNDING``; a test keeps the two equal). A caller +#: gives it as the status of a group or cell the profiler's own rounding withheld. +WITHHELD_ROUNDING = "rounding" + + +@dataclass(frozen=True) +class EdgeStatistic: + """One statistic of a column that counts records between edges, as measured. + + ``identifier`` is the definition's (``marginalHistogram/1``, ``levelHistogram/1``, + ``quantile/1`` or ``quantile/2``). ``edges`` are the encoded bin edges of a + histogram or a ``quantile/2``, ``levels`` a level histogram's declared levels, + ``groups`` the registry's measured groups (:attr:`MeasuredGroup.statistic`) and + ``left_out`` the rows a ``groupBy`` null leaves out of every group, measured as one + more group that is checked and never written. ``group_status`` and ``cell_status`` + hold what the profiler's own checks already withheld, by group key and per cell: + ``F`` or :data:`WITHHELD_ROUNDING`. + """ + + identifier: str + edges: Tuple[int, ...] + levels: Tuple[str, ...] + groups: Tuple[Any, ...] + group_by: Optional[str] = None + level: Optional[Fraction] = None + left_out: Any = None + group_status: Mapping[str, str] = field(default_factory=dict) + cell_status: Mapping[str, Tuple[str, ...]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class EdgeDecision: + """What the check for counts at edges decided about one :class:`EdgeStatistic`. + + ``group_status`` is ``F`` or ``C`` for each group key (the row left out is not + among them) and ``cell_status`` the same per cell of each published histogram + group. ``group_reason`` and ``cell_reason`` say why each withheld value is + withheld, in the registry's words, and are for the profiler's report alone. + ``rounded`` holds each published quantile group spelled as the profile writes + it, its counts already rounded. + """ + + group_status: Mapping[str, str] + cell_status: Mapping[str, Tuple[str, ...]] + group_reason: Mapping[str, str] + cell_reason: Mapping[str, Tuple[str, ...]] + rounded: Mapping[str, Dict[str, Any]] + + @runtime_checkable class MetricDefinition(Protocol): """One registered column metric (``count/1`` ... ``quantiles/1``). @@ -313,8 +372,22 @@ def measure( record the block's declaration refuses. """ - def quantile_ranks_ok(self, k: int, level: Fraction, below: int, at_or_below: int) -> bool: - """Whether a witness with these ranks, over *k* records, is the *level* quantile.""" + def check_edge_counts( + self, + statistics: Sequence[EdgeStatistic], + *, + total: int, + minimum_cell: int, + zeros_disclosive: bool, + round_count: Callable[[int], int], + ) -> Tuple[EdgeDecision, ...]: + """Apply the output check for counts at a column's edges to all its statistics at once. + + *statistics* are every histogram and quantile of one column, in declaration + order, and *total* the column's non-null record count. The result holds one + decision for each. Raises :class:`ValueError` for counts that cannot describe + one set of records. + """ def read_budget(self) -> Any: """A fresh budget for reading one bundle's documents, to share across its files.""" @@ -418,7 +491,7 @@ class _FluxStatsBackend: def __init__(self) -> None: # The package first: when it is missing, this is the import that fails, # even with one of its submodules still in sys.modules. - from flux_stats import Parameters, canonical, definitions, exact_yaml, metrics + from flux_stats import Parameters, canonical, definitions, edge_counts, exact_yaml, metrics from flux_stats.errors import FluxError from flux_stats.fixedpoint import Grid from flux_stats.profile import load_profile_text @@ -432,6 +505,7 @@ def __init__(self) -> None: self._category = category self._read_exact = read_exact self._value_domain = ValueDomain + self._edge_counts = edge_counts self._exact_yaml = exact_yaml self._canonical = canonical self._columns: Dict[Tuple[str, str], _Column] = {} @@ -508,8 +582,7 @@ def _fidelity(self) -> Any: # By name, so a flux-stats that lacks the module raises here. fidelity = import_module("flux_stats.fidelity") reading = import_module("flux_stats.reading") - from flux_stats.fixedpoint import quantile_rank_ok - from flux_stats.registry import definition_for_kind + from flux_stats.registry import definition as definition_named from flux_stats.spelling import render_exact except ImportError as error: missing = error.name or "flux_stats.fidelity" @@ -520,8 +593,7 @@ def _fidelity(self) -> Any: self._grammar = SimpleNamespace( fidelity=fidelity, reading=reading, - quantile_rank_ok=quantile_rank_ok, - definition_for_kind=definition_for_kind, + definition_named=definition_named, render_exact=render_exact, ) return self._grammar @@ -573,6 +645,7 @@ def _declared(grammar: Any, parsed: Any, declaration: Any) -> DeclaredStatistic: column: str = declaration.columns[0] group_by: Optional[str] = declaration.group_by shape: Dict[str, Any] = {} + edges: Tuple[int, ...] = () categorical = kind in ("support", "levelHistogram") if kind == "marginalHistogram": shape["bins"] = [grammar.render_exact(edge) for edge in declaration.bins] @@ -580,18 +653,26 @@ def _declared(grammar: Any, parsed: Any, declaration: Any) -> DeclaredStatistic: shape["levels"] = list(grammar.reading.level_cells(parsed.columns[column])) elif kind == "quantile": shape["level"] = grammar.render_exact(declaration.level) + if declaration.bins: # a quantile that declares bins is quantile/2 + shape["bins"] = [grammar.render_exact(edge) for edge in declaration.bins] + if declaration.bins and kind in ("marginalHistogram", "quantile"): + # As the fidelity check places them: it forms a group on these edges. + edges = tuple( + parsed.grid.quantise(edge, subject="bin edge") for edge in declaration.bins + ) names = (column,) if group_by is None else (column, group_by) read_as = {column: "category" if categorical else parsed.columns[column].domain.value} if group_by is not None: read_as[group_by] = "category" return DeclaredStatistic( - identifier=grammar.definition_for_kind(declaration.kind).identifier, + identifier=grammar.fidelity.definition_for_property(declaration).identifier, kind=kind, column=column, group_by=group_by, shape=shape, columns=names, read_as=read_as, + edges=edges, _declaration=declaration, ) @@ -651,13 +732,18 @@ def _measured( rows, indices, key, declaration, parsed.columns, parsed.grid ) kind = statistic.kind - spelled: Dict[str, Any] = state.to_document(kind) - group = MeasuredGroup(key=key, document=spelled, k=state.k) + # By definition, not by kind: a kind spells a quantile as quantile/1, and + # refuses a group a quantile/2 formed. + definition = grammar.fidelity.definition_for_property(declaration) + spelled: Dict[str, Any] = state.to_document(definition) + group = MeasuredGroup(key=key, document=spelled, k=state.k, statistic=state) if kind in ("marginalHistogram", "levelHistogram"): return replace(group, cells=tuple(state.cell_counts(kind))) if kind == "support": return replace(group, member_holders=tuple(state.support_counts)) if kind == "quantile": + if definition.binned: + return group ranks = (state.quantile_witness, state.quantile_below, state.quantile_at_or_below) return replace(group, quantile=ranks) # A mean: the same indices, read as the block reads them, so the null policy @@ -672,15 +758,81 @@ def _measured( ) return replace(group, values=tuple(values), encoded=tuple(encoded)) - def quantile_ranks_ok(self, k: int, level: Fraction, below: int, at_or_below: int) -> bool: + def check_edge_counts( + self, + statistics: Sequence[EdgeStatistic], + *, + total: int, + minimum_cell: int, + zeros_disclosive: bool, + round_count: Callable[[int], int], + ) -> Tuple[EdgeDecision, ...]: + edges = self._edge_counts grammar = self._fidelity() - try: - proved: bool = grammar.quantile_rank_ok( - count=k, level=level, count_strictly_below=below, count_at_or_below=at_or_below + asked = [] + for each in statistics: + groups = tuple(each.groups) + group_status = dict(each.group_status) + if each.left_out is not None: + # The row a groupBy null leaves out: counted, never published, and under a + # key no declared level can be. + groups += (replace(each.left_out, group=_LEFT_OUT_KEY),) + group_status[_LEFT_OUT_KEY] = _LEFT_OUT_STATUS + asked.append( + edges.EdgeStatistic( + identifier=each.identifier, + edges=each.edges, + levels=each.levels, + groups=groups, + group_by=each.group_by, + level=each.level, + group_status=group_status, + cell_status=dict(each.cell_status), + ) ) - except self._refused: - return False - return proved + decisions = edges.check_edge_counts( + asked, + total=total, + minimum_cell=minimum_cell, + zeros_disclosive=zeros_disclosive, + round_count=round_count, + ) + return tuple( + self._decided(grammar, each, decision) + for each, decision in zip(statistics, decisions, strict=True) + ) + + @staticmethod + def _decided(grammar: Any, each: EdgeStatistic, decision: Any) -> EdgeDecision: + """The registry's decision without the row left out, with each quantile group spelled.""" + definition = grammar.definition_named(each.identifier) + + def kept(values: Mapping[str, Any]) -> Dict[str, Any]: + return {key: value for key, value in values.items() if key != _LEFT_OUT_KEY} + + return EdgeDecision( + group_status=kept(decision.group_status), + cell_status=kept(decision.cell_status), + group_reason=kept(decision.group_reason), + cell_reason=kept(decision.cell_reason), + rounded={ + key: _spelled(group, definition) + for key, group in decision.rounded.items() + if key != _LEFT_OUT_KEY + }, + ) + + +#: The key and status the row a ``groupBy`` null leaves out is checked under: no +#: declared level can start with a NUL, and the status is never ``F``. +_LEFT_OUT_KEY = "\x00left out" +_LEFT_OUT_STATUS = "left out" + + +def _spelled(group: Any, definition: Any) -> Dict[str, Any]: + """A rounded quantile group as the profile writes it.""" + document: Dict[str, Any] = group.to_document(definition) + return document #: Why a correlation is not answered (founder decision, 2026-11-18). diff --git a/fluid_build/profiler/disclosure.py b/fluid_build/profiler/disclosure.py index 04c5525f..44106fe7 100644 --- a/fluid_build/profiler/disclosure.py +++ b/fluid_build/profiler/disclosure.py @@ -29,23 +29,18 @@ also gives the total of what the values lose below their grid points, so that total is checked as a sum as well (:func:`check_mean`). -A minimum, a maximum or a quantile is one record's value in most data. The -DwB/ESSnet "Guidelines for Output Checking" (class 3, maxima, minima and -percentiles) do not release one by rule of thumb, and clear one when more than the -threshold of units hold it. The profiler takes that reading: such a value is -published only when at least ``minimumCell`` records hold exactly that value -(founder decision, 2026-11-16). A quantile's two ranks are counts too, so the -records below its witness and the records above it must each be at least -``minimumCell`` (FLUX ADR 0012's three cells, founder decision 2026-11-18): a rank -of 3 would say that three records lie below the witness. None is allowed only where -zeros are not disclosive, since it says no record lies between the witness and the -end of the range. A published minimum or maximum says that no record lies beyond it, -and no check withholds it for that (FLUX ADR 0012, amended 2026-11-18). This leaves -the checks inconsistent at the ends of the range: where zeros are disclosive, a -quantile whose witness is the column's smallest value is withheld because no record -lies below the witness, though a published minimum already shows that, and one whose -witness is the largest value is withheld because no record lies above it, though a -published maximum already shows that. +A minimum, a maximum or a column's ``quantiles`` metric is one record's value in +most data. The DwB/ESSnet "Guidelines for Output Checking" (class 3, maxima, +minima and percentiles) do not release one by rule of thumb, and clear one when +more than the threshold of units hold it. The profiler takes that reading: such a +value is published only when at least ``minimumCell`` records hold exactly that +value (founder decision, 2026-11-16). A published minimum or maximum says that no +record lies beyond it, and no check withholds it for that (FLUX ADR 0012, amended +2026-11-18). This leaves the checks inconsistent at the ends of the range: where +zeros are disclosive, a quantile whose witness is the column's smallest value is +withheld because no record lies below the witness, though a published minimum +already shows that, and one whose witness is the largest value is withheld because +no record lies above it, though a published maximum already shows that. A verdict names the check that fired with an SDMX ``CL_CONF_STATUS`` 1.4 code: ``A`` for a small count (a zero cell, a cell below the threshold, or a value too @@ -55,20 +50,26 @@ another cannot be recovered (secondary suppression, :func:`suppress_secondary`). A withheld value is written with that code and no value, except in a count table or a quantile (:data:`ONE_CODE`, PROFILE-SPEC §3), whose withheld groups and cells -are all written ``C``, whatever withheld them (:func:`written_status`, FLUX ADR -0012): a code that named the check would tell a reader a range, ``A`` that a cell -holds fewer than ``minimumCell`` records, and a total the reader knows can turn -the range into the count. The profiler's summary still counts them by the check, as -ACRO's outcome table keeps each suppressed cell's reasons apart from the table it -releases. - -The groups of a statistic the Simulation's fidelity block declares are checked -group by group, each as a cell of a table of records (the ESSnet/DwB output +are all written ``C``, whatever withheld them: a code that named the check would +tell a reader a range, ``A`` that a cell holds fewer than ``minimumCell`` records, +and a total the reader knows can turn the range into the count. The profiler's +summary still counts them by the check, as ACRO's outcome table keeps each +suppressed cell's reasons apart from the table it releases. + +The groups of a mean or a support the Simulation's fidelity block declares are +checked group by group, each as a cell of a table of records (the ESSnet/DwB output checking guidelines' frequency-table and magnitude-table classes, which ACRO's ``safe_threshold`` and ``safe_nk`` checks implement): :func:`check_group_count` -for the group's record count, :func:`publish_cells` for a histogram's cells, -:func:`check_support` for a support's members and :func:`check_quantile` for a -quantile's witness. +for the group's record count, :func:`check_support` for a support's members. + +The counts a histogram, a level histogram or a quantile publishes at the edges of +its column are not checked here. FLUX ADR 0012 sets one rule for all of them, which +flux-stats applies (``flux_stats.edge_counts.check_edge_counts``, called through +:mod:`fluid_build.profiler._stats`): one declared partition, three cells for a +quantile, a group's size, no small cell a reader can form by adding or subtracting +published counts, and only then the rounding. What it leaves readable is its own to +say. The profiler passes it every such statistic of a column in one call, with this +module's :func:`round_count`. What the group checks leave readable, on purpose: @@ -80,13 +81,6 @@ it is withheld too, under ``D``, so the withheld one cannot be recovered by differencing against the ungrouped figures. The records a ``groupBy`` null leaves out of every group count as one more group there, never written. -* A group withheld for its size withholds every group of its groupBy's - histograms and quantiles too (the runner's sibling rule, FLUX ADR 0012), and a - quantile is published only as the one histogram or quantile of its column. - A withheld cell of a histogram is not secondarily suppressed: the rounded - counts still bound it, somewhere inside 0 to 9 records and never above the - threshold, until forge-cli applies FLUX's ``check_edge_counts``, which a - flux-stats release has to carry first. Its ``C`` adds nothing to that bound. Where this differs from ACRO 1.1.0, on purpose: @@ -133,7 +127,6 @@ "SECONDARY", "SMALL_COUNT", "WITHHELD", - "Cells", "DisclosureRule", "Exact", "Verdict", @@ -142,15 +135,12 @@ "check_magnitude", "check_mean", "check_order_statistic", - "check_quantile", "check_rounding", "check_sum", "check_support", - "publish_cells", "round_count", "sample_size", "suppress_secondary", - "written_status", ] # SDMX ``CL_CONF_STATUS`` 1.4 codes, with PROFILE-SPEC §3's paraphrase of each @@ -174,27 +164,12 @@ SECONDARY = "D" #: The declared statistics whose every withheld group and cell is written ``C``, whatever -#: withheld it (:func:`written_status`; PROFILE-SPEC §3): the count tables, a histogram -#: over bins and one over levels, and a quantile. +#: withheld it (PROFILE-SPEC §3; the runner writes them from the decisions of flux-stats' +#: ``check_edge_counts``): the count tables, a histogram over bins and one over levels, +#: and a quantile. ONE_CODE = frozenset({"marginalHistogram", "levelHistogram", "quantile"}) -def written_status(status: str, kind: str) -> str: - """The status a document writes for a value that a check gave *status*. - - *kind* is the declared property the value belongs to (``mean``, ``quantile``, - ``marginalHistogram`` and so on). A value of a count table or a quantile that - any check withholds is written ``C``, so its status says nothing about its - size (FLUX ADR 0012, PROFILE-SPEC §3): ``A`` would say that a cell holds fewer - than ``minimumCell`` records, and cells of 10, 9, 9 and 10 records written - ``F``, ``A``, ``A``, ``F`` give both withheld cells as 9 to a reader who knows - the total. Every other value keeps the code of the check that withheld it. - """ - if status != FREE and kind in ONE_CODE: - return WITHHELD - return status - - #: The largest count a profile can carry: 2^53 - 1, as RFC 8785 serialises it. MAX_COUNT = 2**53 - 1 @@ -519,46 +494,6 @@ def check_group_count(k: int, rule: DisclosureRule) -> Verdict: return verdict -@dataclass(frozen=True) -class Cells: - """A histogram group's cells after the checks. - - ``verdict`` is :data:`PUBLISHED`, or ``A`` when every cell is withheld and the - group goes whole. ``statuses`` holds the code of the check that decided each - cell, which a document writes through :func:`written_status`, and ``counts`` - the published count of each, rounded, or ``None`` where the cell is withheld - (PROFILE-SPEC §3: a cell is null exactly when its status is not ``F``). - """ - - verdict: Verdict - statuses: Tuple[str, ...] - counts: Tuple[Optional[int], ...] - - @property - def any_withheld(self) -> bool: - return any(status != FREE for status in self.statuses) - - -def publish_cells(counts: Sequence[int], rule: DisclosureRule) -> Cells: - """Check each cell of a histogram group as a count, and round the ones that pass. - - A cell is a frequency-table cell: a zero is withheld when the rule holds zeros - disclosive, one below ``minimumCell`` is withheld, and so is a positive count - that rounds to zero (:func:`check_count`, then :func:`check_rounding`), each - with ``A``. A published cell is rounded to a multiple of the rounding base. A - group whose every cell is withheld says nothing, so it is withheld whole. - """ - statuses: List[str] = [] - shown: List[Optional[int]] = [] - for count in counts: - verdict = check_group_count(count, rule) - statuses.append(verdict.status) - shown.append(round_count(count, rule) if verdict.published else None) - if statuses and all(status != FREE for status in statuses): - return Cells(Verdict(SMALL_COUNT, "every cell is withheld"), tuple(statuses), tuple(shown)) - return Cells(PUBLISHED, tuple(statuses), tuple(shown)) - - def check_support(holders: Sequence[int], rule: DisclosureRule) -> Verdict: """Check a support: every member must be held by at least ``minimumCell`` records. @@ -576,72 +511,6 @@ def check_support(holders: Sequence[int], rule: DisclosureRule) -> Verdict: return PUBLISHED -def check_quantile( - k: int, - below: int, - at_or_below: int, - rule: DisclosureRule, - proves: Callable[[int, int, int], bool], -) -> Tuple[Verdict, int, int]: - """Check a quantile group's witness and its ranks; return the verdict and the rounded ranks. - - *below* and *at_or_below* are the records strictly below the witness and at or - below it, so ``at_or_below - below`` records hold it and ``k - at_or_below`` - lie above it. In order, the first that fires decides, always with ``A``: - - 1. fewer than ``minimumCell`` records hold the witness - (:func:`check_order_statistic`, as a minimum or a maximum is checked); - 2. the records below the witness, or above it, are fewer than ``minimumCell``: - on unrounded counts, the three cells the ranks state each hold at least - ``minimumCell`` records (FLUX ADR 0012's three cells), since a rank of 3 says - that three records lie below the witness, rounded or not. None below or - above is allowed only where the rule does not hold zeros disclosive; - 3. a rank is positive and rounds to zero (:func:`check_rounding`); - 4. the rounded ``k`` and ranks no longer prove the witness: *proves* is the - registry's rank check on them, because a published group is read back with - the rounded numbers, and a rank rounded across the witness's index would - publish a witness the numbers contradict. - - The ranks are published rounded, as every count is. The returned ranks are - only meaningful when the verdict is published. - """ - for count in (k, below, at_or_below): - _require_count(count) - if not below <= at_or_below <= k: - raise ValueError("a quantile's ranks lie from 0 to its k, the lower one first") - verdict = check_order_statistic(at_or_below - below, rule) - if verdict.published: - verdict = _check_outer_cells(below, k - at_or_below, rule) - if verdict.published: - for rank in (below, at_or_below): - verdict = check_rounding(rank, rule) - if not verdict.published: - break - shown = (round_count(below, rule), round_count(at_or_below, rule)) - if verdict.published and not proves(round_count(k, rule), shown[0], shown[1]): - verdict = Verdict(SMALL_COUNT, "the rounded ranks no longer prove the witness") - return verdict, shown[0], shown[1] - - -def _check_outer_cells(below: int, above: int, rule: DisclosureRule) -> Verdict: - """The records below a quantile's witness and above it: each enough, or none where allowed. - - None below or above says that no record lies between the witness and the end of - the range, a cell of no records, so it is allowed only where the rule does not hold - zeros disclosive (FLUX ADR 0012). - """ - for count, side in ((below, "below"), (above, "above")): - if count == 0 and rule.zeros_disclosive: - return Verdict( - SMALL_COUNT, f"no record lies {side} the value, and the rule holds zeros disclosive" - ) - if 0 < count < rule.minimum_cell: - return Verdict( - SMALL_COUNT, f"fewer than {rule.minimum_cell} records lie {side} the value" - ) - return PUBLISHED - - def suppress_secondary(verdicts: Sequence[Verdict]) -> Tuple[Verdict, ...]: """Withhold every published group of a statistic when any of its groups is withheld. diff --git a/fluid_build/profiler/runner.py b/fluid_build/profiler/runner.py index f233c197..1340173d 100644 --- a/fluid_build/profiler/runner.py +++ b/fluid_build/profiler/runner.py @@ -18,8 +18,9 @@ With a bundle (:mod:`fluid_build.profiler.bundle`) it forms, besides, each statistic the Simulation's fidelity block declares (a histogram, a support, a quantile, a grouped mean), from the same one read of the table. -4. :mod:`fluid_build.profiler.disclosure` decides what each metric and statistic - may publish, group by group. +4. :mod:`fluid_build.profiler.disclosure` decides what each metric, mean and support + may publish, group by group, and the registry's ``check_edge_counts`` (FLUX ADR + 0012) what the histograms and quantiles of one column may publish together. 5. :mod:`fluid_build.profiler.document` writes the canonical document, the registry reads it back, and its digest is taken. @@ -42,9 +43,12 @@ from fluid_build.policy._common import iter_exposes from fluid_build.profiler import disclosure, document, sources from fluid_build.profiler._stats import ( + WITHHELD_ROUNDING, BlockRefused, DeclaredBlock, DeclaredStatistic, + EdgeDecision, + EdgeStatistic, MeasuredGroup, MeasuredStatistic, MeasureRefused, @@ -172,8 +176,8 @@ class ProfileResult: plan: ExposePlan published: int #: Withheld values per ``CL_CONF_STATUS`` code of the check that withheld them: a - #: histogram's withheld groups are written ``C`` whatever the check - #: (:func:`disclosure.written_status`), and counted here by the check. + #: histogram's or a quantile's withheld groups are written ``C`` whatever the check, and + #: counted here by the check (:func:`_code`). withheld: Mapping[str, int] = field(default_factory=dict) #: The histogram cells withheld inside published groups, per code of the check #: that withheld them; the document writes each ``C``. @@ -628,8 +632,8 @@ def _declare_statistics( The rows of each statistic are built from the one read of the table. A statistic no record forms a group of (a grouped one over an empty table) has nothing to write, and is returned with the reason. Every statistic is measured - before any is checked, since a quantile's and a grouped histogram's publication - depend on the column's other statistics (:func:`_withheld_beside`). *tallies* + before any is checked, since what a histogram or a quantile may publish depends on + the column's other statistics (:func:`_edge_decisions`). *tallies* count the values and the histogram cells the checks withhold, by the check. """ rows: Dict[Tuple[str, ...], List[Dict[str, Any]]] = {} @@ -657,124 +661,177 @@ def _declare_statistics( left_out.append((statistic.description, "no record forms a group of it")) continue formed.append((statistic, measured)) - beside = _beside(formed, rule) - for statistic, measured in formed: + decided = _edge_decisions(formed, backend, rule) + for position, (statistic, measured) in enumerate(formed): entry = _declared_entry( - statistic, measured, backend, rule, tallies, fractional_bits, beside + statistic, measured, rule, tallies, fractional_bits, decided.get(position) ) columns_doc[statistic.column].setdefault("statistics", []).append(entry) return left_out #: The declared statistics that count records between edges of their column (FLUX ADR -#: 0012): a quantile is published only as the one of them on its column. -_EDGE_COUNTS = frozenset({"marginalHistogram", "levelHistogram", "quantile"}) +#: 0012), in the order a column's call to the output check lists them. +_EDGE_KINDS = ("marginalHistogram", "levelHistogram", "quantile") -@dataclass(frozen=True) -class _Beside: - """What one statistic's groups give back with the column's others (FLUX ADR 0012). +def _code(reason: str) -> str: + """The ``CL_CONF_STATUS`` code the summary tallies a withheld edge count under. - ``crowded`` holds the columns with more than one statistic that counts at their - edges, where a quantile is withheld. ``small`` holds each (column, groupBy) with a - group the size check withholds, the rows no group holds included, whose - histograms and quantiles are then withheld whole. + ``A`` for a small group or cell and for one the rounding withholds, ``D`` for one + withheld so that another cannot be recovered: off the partition, by differencing, + or beside rows no group holds. """ + return disclosure.SMALL_COUNT if reason in _TALLIED_AS_SMALL else disclosure.SECONDARY + - crowded: FrozenSet[str] - small: FrozenSet[Tuple[str, str]] +_TALLIED_AS_SMALL = frozenset({"small", WITHHELD_ROUNDING}) -def _beside( +def _edge_decisions( formed: Sequence[Tuple[DeclaredStatistic, MeasuredStatistic]], + backend: StatisticsBackend, rule: disclosure.DisclosureRule, -) -> _Beside: - """The columns a quantile shares with another count at edges, and the small groupBys.""" - counts = Counter(statistic.column for statistic, _ in formed if statistic.kind in _EDGE_COUNTS) - small = set() - for statistic, measured in formed: - if statistic.group_by is None: +) -> Dict[int, EdgeDecision]: + """What each histogram and quantile may publish, by its position in *formed*. + + The statistics that count at the edges of one column are checked in one call, in + declaration order (FLUX ADR 0012): a statistic checked on its own is judged alone, so + a quantile would pass beside a histogram and no count would be checked against + another. Where a grouped one has rows no group holds that cannot be read as the block + reads them, the check cannot see what its groups give back, and every group of every + such statistic of the column is withheld instead. + """ + columns: Dict[str, List[int]] = {} + for position, (statistic, _) in enumerate(formed): + if statistic.kind in _EDGE_KINDS: + columns.setdefault(statistic.column, []).append(position) + decided: Dict[int, EdgeDecision] = {} + for column, positions in columns.items(): + if any(formed[each][1].left_out_unread for each in positions): + for each in positions: + decided[each] = _withheld_whole(formed[each][1], "rows no group holds are unread") continue - sizes = [group.k for group in measured.groups] - if measured.left_out is not None: - sizes.append(measured.left_out.k) - if measured.left_out_unread or any( - not disclosure.check_group_count(size, rule).published for size in sizes - ): - small.add((statistic.column, statistic.group_by)) - return _Beside( - frozenset(column for column, count in counts.items() if count > 1), frozenset(small) - ) + statistics = [_edge_statistic(*formed[each], rule) for each in positions] + try: + decisions = backend.check_edge_counts( + statistics, + total=_column_total(*formed[positions[0]]), + minimum_cell=rule.minimum_cell, + zeros_disclosive=rule.zeros_disclosive, + round_count=lambda count: disclosure.round_count(count, rule), + ) + except ValueError as exc: + raise ProfileError( + "profile_edge_counts_refused", + f"the counts of column {column!r} at its edges cannot describe one set of " + f"records: {exc}", + {"written": "nothing", "next": "report this: the output check refused its input"}, + ) from None + decided.update(zip(positions, decisions, strict=True)) + return decided -def _withheld_beside( - statistic: DeclaredStatistic, verdicts: List[disclosure.Verdict], beside: _Beside -) -> List[disclosure.Verdict]: - """A histogram's or a quantile's verdicts once the column's other statistics are seen. - - A quantile counts at its witness, an edge no declaration fixes, so beside another - statistic that counts at its column's edges its ranks less theirs would give a - withheld count back. And a group the size check withholds is the column's count - less its siblings' sizes, so every group of its groupBy's histograms and quantiles - goes with it, as a grouped mean's or support's does - (:func:`disclosure.suppress_secondary`). The groups that passed are withheld with - ``D``, and written ``C``. +def _column_total(statistic: DeclaredStatistic, measured: MeasuredStatistic) -> int: + """The records a column's statistics count: its non-null values. + + An ungrouped statistic counts them in its one group, and a grouped one in its groups + and the rows a null ``groupBy`` key leaves out. """ - reason = None - if statistic.kind == "quantile" and statistic.column in beside.crowded: - reason = "another statistic counts at its column's edges" - elif statistic.kind in _EDGE_COUNTS and statistic.group_by is not None: - if (statistic.column, statistic.group_by) in beside.small: - reason = "a group of its groupBy is withheld for its size" - if reason is None: - return verdicts - withheld = disclosure.Verdict(disclosure.SECONDARY, reason) - return [withheld if verdict.published else verdict for verdict in verdicts] + held = sum(group.k for group in measured.groups) + if statistic.group_by is not None and measured.left_out is not None: + held += measured.left_out.k + return held + + +def _edge_statistic( + statistic: DeclaredStatistic, measured: MeasuredStatistic, rule: disclosure.DisclosureRule +) -> EdgeStatistic: + """One histogram or quantile as the output check reads it, unrounded. + + What the profiler's own rounding check already withholds (a positive count that + rounds to zero) is passed as withheld; the check decides every other withholding. + """ + group_status = { + group.key: WITHHELD_ROUNDING + for group in measured.groups + if not disclosure.check_rounding(group.k, rule).published + } + cell_status: Dict[str, Tuple[str, ...]] = {} + if statistic.kind != "quantile": + for group in measured.groups: + statuses = tuple( + ( + disclosure.FREE + if disclosure.check_rounding(count, rule).published + else WITHHELD_ROUNDING + ) + for count in group.cells + ) + if any(status != disclosure.FREE for status in statuses): + cell_status[group.key] = statuses + return EdgeStatistic( + identifier=statistic.identifier, + edges=statistic.edges, + levels=tuple(statistic.shape.get("levels", ())), + groups=tuple(group.statistic for group in measured.groups), + group_by=statistic.group_by, + level=Fraction(statistic.shape["level"]) if statistic.kind == "quantile" else None, + left_out=measured.left_out.statistic if measured.left_out is not None else None, + group_status=group_status, + cell_status=cell_status, + ) + + +def _withheld_whole(measured: MeasuredStatistic, reason: str) -> EdgeDecision: + """Every group withheld, for a reason the output check was not asked about.""" + keys = [group.key for group in measured.groups] + return EdgeDecision( + group_status={key: disclosure.WITHHELD for key in keys}, + cell_status={}, + group_reason={key: reason for key in keys}, + cell_reason={}, + rounded={}, + ) def _declared_entry( statistic: DeclaredStatistic, measured: MeasuredStatistic, - backend: StatisticsBackend, rule: disclosure.DisclosureRule, tallies: Tuple["Counter[str]", "Counter[str]"], fractional_bits: int, - beside: _Beside, + decision: Optional[EdgeDecision], ) -> Dict[str, Any]: """One declared statistic as the document writes it: every group checked, none guessed at. - Each status is written through :func:`disclosure.written_status`, so a histogram's - and a quantile's withheld groups and cells read ``C``; *tallies* count them by the - check. + A histogram's or a quantile's groups come from the *decision* of the output check on + its column, and a mean's and a support's from their own checks. *tallies* count the + groups and the histogram cells that are withheld, by the check. """ tally, cell_tally = tallies groups = measured.groups - checked = [_check_group(statistic, group, backend, rule, fractional_bits) for group in groups] - verdicts = [verdict for verdict, _ in checked] - if statistic.group_by is not None and statistic.kind in _SECONDARY_SUPPRESSION: - verdicts = _suppressed(statistic, verdicts, measured, backend, rule, fractional_bits) - verdicts = _withheld_beside(statistic, verdicts, beside) written: List[Dict[str, Any]] = [] - for group, verdict, (_, published) in zip(groups, verdicts, checked, strict=True): - tally[verdict.status] += 1 - if verdict.published: - if "cellStatus" in published: - checks = published["cellStatus"] - cell_tally.update(code for code in checks if code != disclosure.FREE) - published = { - **published, - "cellStatus": [ - disclosure.written_status(code, statistic.kind) for code in checks - ], - } - written.append({**published, "status": disclosure.FREE}) - else: - written.append( - { - "group": group.key, - "status": disclosure.written_status(verdict.status, statistic.kind), - } - ) + if decision is not None: + for group in groups: + if decision.group_status[group.key] == disclosure.FREE: + tally[disclosure.FREE] += 1 + published = _published_edge_group(statistic, group, decision, rule, cell_tally) + written.append({**published, "status": disclosure.FREE}) + else: + tally[_code(decision.group_reason[group.key])] += 1 + written.append({"group": group.key, "status": disclosure.WITHHELD}) + else: + checked = [_check_group(statistic, group, rule, fractional_bits) for group in groups] + verdicts = [verdict for verdict, _ in checked] + if statistic.group_by is not None: + verdicts = _suppressed(statistic, verdicts, measured, rule, fractional_bits) + for group, verdict, (_, published) in zip(groups, verdicts, checked, strict=True): + tally[verdict.status] += 1 + if verdict.published: + written.append({**published, "status": disclosure.FREE}) + else: + written.append({"group": group.key, "status": verdict.status}) entry: Dict[str, Any] = { "definition": statistic.identifier, "sampleSize": disclosure.sample_size(sum(group.k for group in groups), rule), @@ -787,16 +844,44 @@ def _declared_entry( return entry -#: The declared statistics whose groups partition a total the profile publishes, so one -#: withheld group would be given back by the others (:func:`disclosure.suppress_secondary`). -_SECONDARY_SUPPRESSION = frozenset({"mean", "support"}) +def _published_edge_group( + statistic: DeclaredStatistic, + group: MeasuredGroup, + decision: EdgeDecision, + rule: disclosure.DisclosureRule, + cell_tally: "Counter[str]", +) -> Dict[str, Any]: + """A published group of a histogram or a quantile, its counts rounded. + + A quantile's group is spelled by the registry with its counts already rounded. A + histogram's cells are rounded here, and a cell the check withholds is ``null`` with + status ``C`` in ``cellStatus``, which is written only when some cell is withheld. + """ + if statistic.kind == "quantile": + return dict(decision.rounded[group.key]) + published = dict(group.document) + published["k"] = disclosure.round_count(group.k, rule) + statuses = decision.cell_status[group.key] + reasons = decision.cell_reason[group.key] + field_name = "binCounts" if statistic.kind == "marginalHistogram" else "levelCounts" + published[field_name] = [ + disclosure.round_count(count, rule) if status == disclosure.FREE else None + for count, status in zip(group.cells, statuses, strict=True) + ] + if any(status != disclosure.FREE for status in statuses): + published["cellStatus"] = list(statuses) + cell_tally.update( + _code(reason) + for reason, status in zip(reasons, statuses, strict=True) + if status != disclosure.FREE + ) + return published def _suppressed( statistic: DeclaredStatistic, verdicts: List[disclosure.Verdict], measured: MeasuredStatistic, - backend: StatisticsBackend, rule: disclosure.DisclosureRule, fractional_bits: int, ) -> List[disclosure.Verdict]: @@ -811,23 +896,20 @@ def _suppressed( if measured.left_out_unread: hidden.append(disclosure.Verdict(disclosure.SMALL_COUNT, "rows no group holds")) elif measured.left_out is not None: - hidden.append(_check_group(statistic, measured.left_out, backend, rule, fractional_bits)[0]) + hidden.append(_check_group(statistic, measured.left_out, rule, fractional_bits)[0]) return list(disclosure.suppress_secondary([*verdicts, *hidden]))[: len(verdicts)] def _check_group( statistic: DeclaredStatistic, group: MeasuredGroup, - backend: StatisticsBackend, rule: disclosure.DisclosureRule, fractional_bits: int, ) -> Tuple[disclosure.Verdict, Dict[str, Any]]: - """One group's verdict, and its document as it is published when the verdict allows. + """A mean's or a support's group: its verdict, and its document as published when it allows. Every group first passes as a count of records (``k``), which is published rounded; then what its statistic adds is checked as that statistic's rule says. - A histogram's ``cellStatus`` still names the check that withheld each cell: - :func:`_declared_entry` writes it. """ kind = statistic.kind published = dict(group.document) @@ -837,28 +919,8 @@ def _check_group( verdict = disclosure.check_group_count(group.k, rule) if not verdict.published: return verdict, published - if kind in ("marginalHistogram", "levelHistogram"): - cells = disclosure.publish_cells(group.cells, rule) - field_name = "binCounts" if kind == "marginalHistogram" else "levelCounts" - published[field_name] = list(cells.counts) - if cells.any_withheld: - published["cellStatus"] = list(cells.statuses) - return cells.verdict, published if kind == "support": return disclosure.check_support(group.member_holders, rule), published - if kind == "quantile" and group.quantile is not None: - level = Fraction(statistic.shape["level"]) - _, below, at_or_below = group.quantile - verdict, shown_below, shown_at = disclosure.check_quantile( - group.k, - below, - at_or_below, - rule, - lambda k, lower, upper: backend.quantile_ranks_ok(k, level, lower, upper), - ) - published["quantileBelow"] = shown_below - published["quantileAtOrBelow"] = shown_at - return verdict, published raise AssertionError(f"no disclosure rule for declared statistic {statistic.identifier!r}") @@ -873,11 +935,12 @@ def _row_count(rows: int, rule: disclosure.DisclosureRule, tally: "Counter[str]" def _shape(statistic: Mapping[str, Any]) -> Optional[str]: """What tells two statistics of one column and definition apart in the report.""" + parts = [] if "level" in statistic: - return f"at level {statistic['level']}" + parts.append(f"at level {statistic['level']}") if "bins" in statistic: - return f"over bins {', '.join(statistic['bins'])}" - return None + parts.append(f"over bins {', '.join(statistic['bins'])}") + return " ".join(parts) or None def _statistic_summaries( diff --git a/tests/profiler/test_bundle_flux_stats.py b/tests/profiler/test_bundle_flux_stats.py index 6209920a..c9d5ef41 100644 --- a/tests/profiler/test_bundle_flux_stats.py +++ b/tests/profiler/test_bundle_flux_stats.py @@ -34,7 +34,7 @@ from flux_stats.profile import load_profile_text # noqa: E402 from fluid_build.cli import main # noqa: E402 -from fluid_build.profiler import _stats, bundle, disclosure, runner # noqa: E402 +from fluid_build.profiler import _stats, bundle, disclosure, document # noqa: E402 from fluid_build.profiler._stats import DeclaredStatistic # noqa: E402 from fluid_build.profiler.runner import ProfileError, profile_expose # noqa: E402 from tests.profiler.test_flux_stats_backend import _slice # noqa: E402 @@ -215,11 +215,13 @@ def test_the_document_answers_every_declared_property(tmp_path): levels = _entry(result, "market", "levelHistogram/1") assert levels["levels"] == ["DE", "FR", "GB", "IE"] (level_group,) = levels["groups"] - # DE is declared and no record holds it: a zero cell is withheld, the rest are published. - # A histogram's withheld cell is written C, whatever withheld it; the result keeps why. - assert level_group["levelCounts"] == [None, 40, 40, 40] - assert level_group["cellStatus"] == ["C", "F", "F", "F"] - assert result.withheld_cells == {"A": 1} + # DE is declared and no record holds it: a zero cell is withheld, and with the column's + # count known, the three others would give it back, so FR is withheld with it (a formed + # zero is protected). A histogram's withheld cell is written C, whatever withheld it; + # the result keeps why: DE is small, FR is secondary. + assert level_group["levelCounts"] == [None, None, 40, 40] + assert level_group["cellStatus"] == ["C", "C", "F", "F"] + assert result.withheld_cells == {"A": 1, "D": 1} assert _entry(result, "market", "support/1")["groups"][0]["support"] == ["FR", "GB", "IE"] @@ -400,29 +402,198 @@ def test_a_group_withheld_for_its_size_withholds_its_siblings_in_a_grouped_quant assert result.withheld["D"] == 3 -@pytest.mark.parametrize("kind", ["marginalHistogram", "levelHistogram", "quantile"]) -def test_the_sibling_rule_reaches_every_histogram_and_quantile_of_the_column(kind): - # A level histogram needs a second categorical column to be grouped by, which this - # module's contract lacks, so the rule is checked on its own. - passed = disclosure.PUBLISHED - beside = runner._Beside(crowded=frozenset(), small=frozenset({("market", "tier")})) - - def declared(kind: str, column: str) -> DeclaredStatistic: - return DeclaredStatistic( - identifier=f"{kind}/1", - kind=kind, - column=column, - group_by="tier", - shape={}, - columns=(column, "tier"), - read_as={}, - ) +@pytest.mark.parametrize( + "grouped", + [ + pytest.param({**PROPERTIES[2], "groupBy": "market"}, id="marginalHistogram"), + pytest.param( + {**PROPERTIES[5], "groupBy": "market", "bins": ["0", "50", "100"]}, id="quantile-2" + ), + ], +) +def test_a_small_group_withholds_every_group_of_its_groupby_in_each_statistic(tmp_path, grouped): + rows = _rows() + for row in rows[:3]: + row["market"] = "DE" # a group of three records + result = _profile(tmp_path, rows, [grouped]) + (entry,) = [e for e in _entries(result, "amount") if e.get("groupBy") == "market"] + assert {g["status"] for g in entry["groups"]} == {"C"} + assert all(set(g) == {"group", "status"} for g in entry["groups"]) + + +def _row(amount: str, market: str = "IE") -> Dict[str, Any]: + return {"amount": amount, "market": market, "qty": "1"} + + +def _histogram(bins: List[str], column: str = "amount", **more: Any) -> Dict[str, Any]: + return { + "property": "marginalHistogram", + "column": column, + "bins": bins, + "tolerance": _DISTANCE, + **more, + } + + +def _histograms(result, column: str = "amount") -> List[Dict[str, Any]]: + return [e for e in _entries(result, column) if e["definition"] == "marginalHistogram/1"] + + +def test_a_histogram_and_a_coarser_one_of_a_column_give_no_withheld_cell_back(tmp_path): + # Fine cells of 30, 5, 40 and 45 records: the 5 is small, and the coarse cell of 35 beside + # the 30 would give it back by subtraction. Neither histogram is checked alone. + rows = [_row("10")] * 30 + [_row("30")] * 5 + [_row("60")] * 40 + [_row("80")] * 45 + fine = _histogram(["0", "25", "50", "75", "100"]) + coarse = _histogram(["0", "50", "100"]) + result = _profile(tmp_path, rows, [fine, coarse]) + fine_group, coarse_group = (histogram["groups"][0] for histogram in _histograms(result)) + fine_cells, coarse_cells = fine_group["binCounts"], coarse_group["binCounts"] + assert fine_cells[1] is None # the 5 itself + for cell, inside in ((0, fine_cells[:2]), (1, fine_cells[2:])): + if coarse_cells[cell] is not None: + # A published coarse cell with exactly one withheld cell inside it gives that cell back. + assert sum(count is None for count in inside) != 1 + published = [count for count in fine_cells + coarse_cells if count is not None] + assert 5 not in published and 35 not in published + withheld = [count is None for count in fine_cells] + assert sum(withheld) >= 2 # a single withheld cell is the total less the published ones + assert result.withheld_cells["D"] >= 1 # more than the small cell itself was withheld + load_profile_text(result.canonical.decode("utf-8")) + + +def test_a_level_histogram_with_one_small_level_withholds_a_neighbour(tmp_path): + # Levels DE, FR, GB, IE hold 5, 25, 40 and 50 records. The 5 is small, and the column's + # count less the three published levels is exactly that 5: another level goes with it. + rows = ( + [_row("10", "IE")] * 50 + + [_row("10", "GB")] * 40 + + [_row("10", "FR")] * 25 + + [_row("10", "DE")] * 5 + ) + result = _profile(tmp_path, rows, [PROPERTIES[3]]) + (group,) = _entry(result, "market", "levelHistogram/1")["groups"] + assert group["cellStatus"][0] == "C" # DE, the small level + assert group["cellStatus"].count("C") == 2 + assert group["levelCounts"][0] is None and None in group["levelCounts"][1:] + assert result.withheld_cells == {"A": 1, "D": 1} - withheld = runner._withheld_beside(declared(kind, "market"), [passed, passed], beside) - assert [verdict.status for verdict in withheld] == ["D", "D"] - # The same groupBy on another column, and a mean, are left to their own checks. - assert runner._withheld_beside(declared(kind, "channel"), [passed], beside) == [passed] - assert runner._withheld_beside(declared("mean", "market"), [passed], beside) == [passed] + +def test_a_formed_zero_is_protected_under_the_default_rule(tmp_path): + # Cells of 12, 0 and 12 records: the zero is withheld (ACRO's default holds zeros + # disclosive), and the column's 24 less the two published 12s would give it back. + rows = [_row("5")] * 12 + [_row("25")] * 12 + result = _profile(tmp_path, rows, [_histogram(["0", "10", "20", "100"])]) + (group,) = _histograms(result)[0]["groups"] + assert group["cellStatus"][1] == "C" and group["binCounts"][1] is None + assert group["cellStatus"].count("C") == 2 + assert group["binCounts"][0] == 10 or group["binCounts"][2] == 10 # the 12 left, rounded + assert result.withheld_cells == {"A": 1, "D": 1} + + +def test_a_formed_zero_is_published_where_the_rule_holds_zeros_safe(tmp_path): + from dataclasses import replace + + rule = replace(disclosure.ACRO_DEFAULT, name="zeros-safe", zeros_disclosive=False) + directory = _bundle(tmp_path, [_histogram(["0", "10", "20", "100"])]) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) + rows = [_row("5")] * 12 + [_row("25")] * 12 + result = profile_expose( + CONTRACT, + EXPOSE, + source=_slice(tmp_path, rows), + backend=_stats.load_backend(), + cwd=tmp_path, + bundle=block, + rule=rule, + ) + (group,) = _histograms(result)[0]["groups"] + assert group["binCounts"] == [10, 0, 10] and "cellStatus" not in group + + +def test_a_quantile_with_bins_is_written_as_quantile_2(tmp_path): + binned = {**PROPERTIES[5], "bins": ["0", "50", "100"]} + result = _profile(tmp_path, _rows(), [binned]) + entry = _entry(result, "amount", "quantile/2") + assert entry["level"] == "0.5" and entry["bins"] == ["0", "50", "100"] + # 120 records, the median is the 60th, a 40: in the first bin, with 72 records through it + # (rounded to 70), none below it, and 48 above it. + assert entry["groups"] == [ + {"group": "", "k": 120, "quantileBin": 0, "binBelow": 0, "binThrough": 70, "status": "F"} + ] + assert "quantileWitness" not in json.dumps(entry) + assert result.document["_schemaURL"] == document.SCHEMA_URL_1_1_0 + profile, _ = load_profile_text(result.canonical.decode("utf-8")) + (read,) = [ + e for e in profile.column_statistics(EXPOSE, "amount") if e.identifier == "quantile/2" + ] + assert read.bins == (Fraction(0), Fraction(50), Fraction(100)) + assert read.level == Fraction(1, 2) + (group,) = read.groups + assert group.published and group.statistic.quantile_bin == 0 + assert (group.statistic.bin_below, group.statistic.bin_through) == (0, 70) + assert result.statistics.count( + ("amount", "quantile/2", None, "at level 0.5 over bins 0, 50, 100") + ) + + +def test_the_engines_matcher_finds_a_quantile_with_bins(tmp_path): + binned = {**PROPERTIES[5], "bins": ["0", "50", "100"]} + result = _profile(tmp_path, _rows(), [binned]) + profile, _ = load_profile_text(result.canonical.decode("utf-8")) + expose = expose_from_contract(CONTRACT, EXPOSE) + parsed = parse_fidelity_block(_block([binned]), expose=expose) + from flux_stats.fidelity import definition_for_property + + (declaration,) = parsed.properties + found = [ + entry + for entry in profile.column_statistics(EXPOSE, "amount") + if entry.identifier == definition_for_property(declaration).identifier + and entry.bins == declaration.bins + and entry.level == declaration.level + ] + assert len(found) == 1 + + +def test_a_profile_without_a_quantile_with_bins_keeps_format_1_0_0(tmp_path): + result = _profile(tmp_path, _rows()) + assert result.document["_schemaURL"] == document.SCHEMA_URL + assert "quantile/2" not in result.canonical.decode("utf-8") + + +def test_a_quantile_with_bins_beside_a_histogram_on_its_partition_is_published(tmp_path): + # The bins of the quantile are the histogram's: one partition, so it is not withheld + # for standing beside another count at the column's edges, as a quantile/1 is. + binned = {**PROPERTIES[5], "bins": ["0", "50", "100"]} + result = _profile(tmp_path, _rows(), [PROPERTIES[2], binned]) + (group,) = _entry(result, "amount", "quantile/2")["groups"] + assert group["status"] == "F" + assert _histograms(result)[0]["groups"][0]["status"] == "F" + + +def test_rows_no_group_holds_are_one_more_group_the_check_counts_and_never_writes(tmp_path): + rows = _rows() + for row in rows[:15]: + row["market"] = None # fifteen records in no group, and in the column's count + by_market = {**PROPERTIES[2], "groupBy": "market"} + result = _profile(tmp_path, rows, [by_market], market_null="exclude") + grouped = _entry(result, "amount", "marginalHistogram/1", "market") + assert [g["group"] for g in grouped["groups"]] == ["FR", "GB", "IE"] + assert {g["status"] for g in grouped["groups"]} == {"F"} + assert b"\\u0000" not in result.canonical and b"\x00" not in result.canonical + + +def test_a_refusal_of_the_counts_by_the_output_check_stops_the_profile_naming_no_value( + tmp_path, monkeypatch +): + def refuse(self, statistics, **_kwargs): + raise ValueError("the groups of 'market' do not add up to the column's total") + + monkeypatch.setattr(_stats._FluxStatsBackend, "check_edge_counts", refuse) + with pytest.raises(ProfileError) as refused: + _profile(tmp_path, _rows(), [PROPERTIES[2]]) + assert refused.value.event == "profile_edge_counts_refused" + assert "column 'amount'" in str(refused.value) def test_a_quantile_beside_a_histogram_of_another_column_is_published(tmp_path): @@ -538,7 +709,7 @@ def test_two_quantiles_of_one_column_are_named_apart_by_level(tmp_path): assert shapes == {"at level 0.5", "at level 0.25"} -def test_a_zero_cell_is_withheld_and_the_group_published(tmp_path): +def test_a_zero_cell_is_withheld_with_the_cell_that_gives_it_back(tmp_path): rows = [{"amount": "10", "market": "IE", "qty": "1"} for _ in range(120)] properties = [ { @@ -550,8 +721,9 @@ def test_a_zero_cell_is_withheld_and_the_group_published(tmp_path): ] result = _profile(tmp_path, rows, properties) (group,) = _entry(result, "amount", "marginalHistogram/1")["groups"] - assert group["binCounts"] == [None, 120] and group["cellStatus"] == ["C", "F"] - assert result.withheld_cells == {"A": 1} + # The zero cell is withheld, and the 120 beside it would give it back: it goes too. + assert group["binCounts"] == [None, None] and group["cellStatus"] == ["C", "C"] + assert result.withheld_cells == {"A": 1, "D": 1} def test_a_small_table_withholds_the_histogram_whole(tmp_path): @@ -684,14 +856,6 @@ def test_the_seam_deduplicates_identical_properties(tmp_path): assert [name for name, _ in declared.left_out] == ["correlation of 'amount' and 'qty'"] -def test_quantile_ranks_are_proved_by_the_registry(tmp_path): - backend = _stats.load_backend() - # 120 records, the median is the 60th: 48 strictly below, 72 at or below proves it. - assert backend.quantile_ranks_ok(120, Fraction(1, 2), 48, 72) - assert not backend.quantile_ranks_ok(120, Fraction(1, 2), 60, 70) - assert not backend.quantile_ranks_ok(120, Fraction(1, 2), 70, 45) # not a sub-count - - def test_the_command_writes_the_document_and_names_what_it_answers(tmp_path, monkeypatch, capsys): directory = _bundle(tmp_path, PROPERTIES) monkeypatch.chdir(tmp_path) @@ -714,8 +878,12 @@ def test_the_command_writes_the_document_and_names_what_it_answers(tmp_path, mon assert "levelHistogram/1 of 'market'" in out and "quantile/1 of 'amount'" in out assert "Not answered: correlation of 'amount' and 'qty':" in out assert "a run that checks it is refused (FLX-5037)." in out - # The DE level holds no record: its cell is withheld, written C, and counted by its check. - assert "Withheld histogram cells, by the check (CL_CONF_STATUS code): A (small count) 1." in out + # The DE level holds no record: its cell is withheld, written C, and counted by its check, + # and FR goes with it so that the count cannot be given back. + assert ( + "Withheld histogram cells, by the check (CL_CONF_STATUS code): " + "A (small count) 1, D (secondary suppression) 1." in out + ) assert _ONE_CODE in out written = (tmp_path / "payment.profile.json").read_bytes() load_profile_text(written.decode("utf-8")) @@ -838,32 +1006,32 @@ def test_the_report_names_the_statistics_written_with_every_group_withheld( def _partly_rows() -> List[Dict[str, Any]]: rows = _rows() for row in rows[:12]: - # Twelve records, 8 and 4 in the two bins: every cell is small, so the group is - # withheld, and its size is not, so the other groups are published. + # Twelve records in one group, 8 and 4 in the two bins: the cell that holds the + # median is small, so that group is withheld, and its size is not, so the other + # groups are published. row["market"] = "DE" return rows _PARTLY_PROPERTIES = [ { - "property": "marginalHistogram", + "property": "quantile", "column": "amount", "groupBy": "market", + "level": "0.5", "bins": ["0", "50", "100"], - "tolerance": _DISTANCE, + "tolerance": _ABSOLUTE, } ] def test_a_statistic_with_some_groups_withheld_is_named_with_its_counts(tmp_path): result = _profile(tmp_path, _partly_rows(), _PARTLY_PROPERTIES) - statuses = [ - g["status"] for g in _entry(result, "amount", "marginalHistogram/1", "market")["groups"] - ] + statuses = [g["status"] for g in _entry(result, "amount", "quantile/2", "market")["groups"]] published = statuses.count("F") assert 0 < published < len(statuses) assert set(statuses) == {"F", "C"} - named = ("amount", "marginalHistogram/1", "market", "over bins 0, 50, 100") + named = ("amount", "quantile/2", "market", "at level 0.5 over bins 0, 50, 100") assert result.partly_withheld_statistics == ((*named, published, len(statuses)),) assert named not in result.statistics assert named not in result.withheld_statistics @@ -888,7 +1056,8 @@ def test_the_report_names_the_statistics_withheld_in_part(tmp_path, monkeypatch, assert rc == 0, out assert ( "Partly withheld, so a run that checks them scores only the published groups: " - "marginalHistogram/1 of 'amount' over bins 0, 50, 100 by 'market' (3 of 4 groups)." in out + "quantile/2 of 'amount' at level 0.5 over bins 0, 50, 100 by 'market' " + "(3 of 4 groups)." in out ) assert _ONE_CODE in out and "Withheld histogram cells" not in out diff --git a/tests/profiler/test_disclosure.py b/tests/profiler/test_disclosure.py index 7582fdf9..528a5478 100644 --- a/tests/profiler/test_disclosure.py +++ b/tests/profiler/test_disclosure.py @@ -394,37 +394,6 @@ def test_a_k_that_rounds_to_zero_is_withheld_under_a_low_threshold(): assert disclosure.check_group_count(3, rule).status == "F" -def test_cells_are_checked_one_by_one_and_rounded(): - cells = disclosure.publish_cells([0, 3, 12, 40], ACRO_DEFAULT) - assert cells.statuses == ("A", "A", "F", "F") - assert cells.counts == (None, None, 10, 40) - assert cells.any_withheld - assert cells.verdict.published - - -def test_cells_that_all_pass_carry_no_withheld_cell(): - cells = disclosure.publish_cells([10, 25, 31], ACRO_DEFAULT) - assert cells.statuses == ("F", "F", "F") - assert cells.counts == (10, 25, 30) - assert not cells.any_withheld - - -def test_a_group_whose_every_cell_is_withheld_is_withheld_whole(): - cells = disclosure.publish_cells([0, 4, 9], ACRO_DEFAULT) - assert cells.verdict.status == "A" - assert cells.counts == (None, None, None) - - -def test_a_zero_cell_is_published_where_the_rule_holds_zeros_safe(): - cells = disclosure.publish_cells([0, 20], _rule(zeros_disclosive=False)) - assert cells.statuses == ("F", "F") and cells.counts == (0, 20) - - -def test_a_positive_cell_that_rounds_to_zero_is_withheld(): - cells = disclosure.publish_cells([2, 30], _rule(minimum_cell=1)) - assert cells.statuses == ("A", "F") - - @pytest.mark.parametrize( ("holders", "status"), [((10, 12), "F"), ((10, 9), "A"), ((1,), "A"), ((), "F"), ((40, 0), "A")], @@ -433,118 +402,9 @@ def test_a_support_needs_every_member_held_by_enough_records(holders, status): assert disclosure.check_support(holders, ACRO_DEFAULT).status == status -def _proves(k, below, at_or_below, level=Fraction(1, 2)): - index = -((-level.numerator * k) // level.denominator) - return below < index <= at_or_below - - -def test_a_quantile_witness_needs_enough_holders_and_publishes_rounded_ranks(): - verdict, below, at_or_below = disclosure.check_quantile(120, 48, 72, ACRO_DEFAULT, _proves) - assert verdict.status == "F" and (below, at_or_below) == (50, 70) - verdict, *_ = disclosure.check_quantile(120, 48, 57, ACRO_DEFAULT, _proves) - assert verdict.status == "A" # nine records hold the witness - verdict, *_ = disclosure.check_quantile(120, 48, 58, ACRO_DEFAULT, _proves) - assert verdict.status == "F" # ten do - - -def test_a_quantile_whose_rounded_ranks_no_longer_prove_the_witness_is_withheld(): - # 40 records, the median is the 20th. 17 below and 27 at or below prove it, with - # 10 holders and 13 records above, and so do their rounded 15 and 25. - assert _proves(40, 17, 27) - verdict, below, at_or_below = disclosure.check_quantile(40, 17, 27, ACRO_DEFAULT, _proves) - assert (below, at_or_below) == (15, 25) and verdict.status == "F" - # 18 below and 28 at or below prove it too, and round to 20 and 30: 20 records - # below no longer leaves the 20th record in the witness's place. - assert _proves(40, 18, 28) - verdict, below, at_or_below = disclosure.check_quantile(40, 18, 28, ACRO_DEFAULT, _proves) - assert (below, at_or_below) == (20, 30) - assert verdict.status == "A" and "no longer prove" in verdict.reason - - -@pytest.mark.parametrize( - ("k", "below", "at_or_below", "status", "side"), - [ - # The records below the witness, holding it and above it: each at least - # minimumCell (FLUX ADR 0012's three cells). None is a zero, and ACRO's - # default holds zeros disclosive. - (120, 9, 30, "A", "below"), - (120, 1, 30, "A", "below"), - (120, 10, 30, "F", None), - (120, 0, 30, "A", "below"), # the witness is the smallest value - (120, 50, 111, "A", "above"), - (120, 50, 119, "A", "above"), - (120, 50, 110, "F", None), - (120, 50, 120, "A", "above"), # the witness is the largest value - (10, 0, 10, "A", "below"), # every record holds it - ], -) -def test_a_quantile_needs_enough_records_below_and_above_its_witness( - k, below, at_or_below, status, side -): - verdict, *_ = disclosure.check_quantile(k, below, at_or_below, ACRO_DEFAULT, lambda *_: True) - assert verdict.status == status - if side is not None: - assert f"{side} the value" in verdict.reason - - -@pytest.mark.parametrize(("below", "at_or_below"), [(0, 30), (50, 120)]) -def test_a_quantile_at_an_end_is_published_where_zeros_are_not_disclosive(below, at_or_below): - rule = _rule(zeros_disclosive=False) - verdict, *_ = disclosure.check_quantile(120, below, at_or_below, rule, lambda *_: True) - assert verdict.status == "F" - - -def test_the_outer_cells_are_checked_before_rounding(): - # 3 records above would round to 5 and read as a cell the threshold cleared. - verdict, *_ = disclosure.check_quantile(60, 20, 57, ACRO_DEFAULT, lambda *_: True) - assert verdict.status == "A" and "above" in verdict.reason - - -@pytest.mark.parametrize( - ("k", "below", "at_or_below"), [(120, 50, 40), (120, 50, 121), (120, -1, 30)] -) -def test_quantile_ranks_that_cannot_describe_the_group_are_refused(k, below, at_or_below): - with pytest.raises(ValueError): - disclosure.check_quantile(k, below, at_or_below, ACRO_DEFAULT, lambda *_: True) - - -def test_a_quantile_rank_that_rounds_to_zero_withholds_the_group(): - rule = _rule(minimum_cell=1, zeros_disclosive=False) - verdict, *_ = disclosure.check_quantile(30, 2, 20, rule, lambda *_: True) - assert verdict.status == "A" - verdict, *_ = disclosure.check_quantile(30, 0, 20, rule, lambda *_: True) - assert verdict.status == "F" # nothing below the witness is a zero rank, not a small one - - -@pytest.mark.parametrize( - ("status", "kind", "written"), - [ - ("A", "marginalHistogram", "C"), - ("A", "levelHistogram", "C"), - ("D", "levelHistogram", "C"), - ("C", "marginalHistogram", "C"), - ("F", "marginalHistogram", "F"), - ("F", "levelHistogram", "F"), - ("A", "mean", "A"), - ("D", "mean", "D"), - ("A", "support", "A"), - ("A", "quantile", "C"), - ("D", "quantile", "C"), - ("O", "mean", "O"), - ], -) -def test_a_count_table_or_a_quantile_writes_c_for_whatever_withheld_a_value(status, kind, written): - assert disclosure.written_status(status, kind) == written - - -def test_one_withheld_code_hides_the_size_a_total_gives_back(): - # PROFILE-SPEC section 3: cells of 10, 9, 9 and 10 records, written F, A, A, F under a - # known total, give both withheld cells as 9; written F, C, C, F they read as a code - # that names no check. - cells = disclosure.publish_cells([10, 9, 9, 10], ACRO_DEFAULT) - assert cells.statuses == ("F", "A", "A", "F") - written = [disclosure.written_status(code, "levelHistogram") for code in cells.statuses] - assert written == ["F", "C", "C", "F"] +def test_the_counts_a_check_for_edges_writes_are_the_ones_the_summary_names(): + # PROFILE-SPEC section 3: a histogram's or a quantile's withheld groups and cells are all + # written C, so the report says so for exactly these definitions. assert disclosure.ONE_CODE == {"marginalHistogram", "levelHistogram", "quantile"} diff --git a/tests/profiler/test_flux_stats_backend.py b/tests/profiler/test_flux_stats_backend.py index 2f821994..cb1950a5 100644 --- a/tests/profiler/test_flux_stats_backend.py +++ b/tests/profiler/test_flux_stats_backend.py @@ -285,3 +285,20 @@ def test_the_mean_a_profile_carries_is_the_one_the_check_compares(tmp_path, back assert entry.definition.compare(same, reference, check) higher = GroupStatistic(group="", k=reference.k, sum_x=reference.sum_x * 12 // 10) assert not entry.definition.compare(higher, reference, check) + + +def test_the_rounding_status_is_the_registrys_word_for_it(): + from flux_stats import edge_counts + + from fluid_build.profiler._stats import WITHHELD_ROUNDING + + assert WITHHELD_ROUNDING == edge_counts.ROUNDING + + +def test_the_backend_reads_a_bundle_through_the_registrys_exact_reader(): + backend = load_backend() + budget = backend.read_budget() + (document,) = backend.read_documents("rate: 0.50\nwhen: 2024-01-01\n", name="s", budget=budget) + assert document == {"rate": Decimal("0.50"), "when": "2024-01-01"} + assert backend.canonicalise(document, name="s") == {"rate": "0.50", "when": "2024-01-01"} + From 5644e72637e7a22ed3c0d1d288146c5945fed940 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:33:41 +0100 Subject: [PATCH 4/9] docs(profile): say what fluid profile checks across a column's edge counts CHANGELOG and the --help text describe the one rule for histograms, level histograms and quantiles, the quantile/2 a quantile with bins is published as, format 1-1-0, and the bundle reader. The note that the earlier entries did not complete ADR 0012's rule is removed, since this completes it. AGENTS.md names the new seam calls. --- AGENTS.md | 2 +- fluid_build/cli/profile_cmd.py | 54 ++++++++++++++--------- tests/profiler/test_flux_stats_backend.py | 1 - 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 25a0f05c..bec70145 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,7 +164,7 @@ forge-cli/ | Bundle builder | `fluid_build/forge/core/bundle.py` — deterministic tgz + MANIFEST + fragment extraction + `$source` sentinel | | Build runners | `fluid_build/build_runners/{dbt,python}/` — extracted from legacy `cli/execute.py` in Phase 1 | | Masking at landing | `fluid_build/build_runners/_masking.py` — the DuckDB runner applies `exposes[].policy.privacy.masking[]` inside the COPY (file, S3, BigQuery staging, DLQ); secrets only from env (`FLUID_PII_HASH_SECRET`, `FLUID_PII_TOKENIZATION_KEY`, `FLUID_PII_ENCRYPTION_SECRET_KEY`, or `params.saltEnv`/`keyEnv`), held in UDF closures, never in SQL; an unappliable rule refuses the build, and so does a DuckDB without numpy (its `create_function` needs it; `[local]` names numpy, and the refusal names the extra). `cli/_verify_masking.py` is stage 9's shape check (local DuckDB, Athena in the count query) | -| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads one expose's records where they live (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC 1-0-0 document (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). With `--bundle DIR` it also answers the fidelity block of DIR's one Simulation: `bundle.py` reads `*.flux.yml` as the engine does (its exact YAML loader, then KERNEL-SPEC 2.1 canonical form), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | +| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads one expose's records where they live (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC 1-0-0 document (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). With `--bundle DIR` it also answers the fidelity block of DIR's one Simulation: `bundle.py` reads `*.flux.yml` as the engine does (flux-stats' `flux_stats.exact_yaml` and `flux_stats.canonical`, through the seam, one `ReadBudget` per bundle), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a quantile with bins is `quantile/2` (the profile then names format 1-1-0), the histograms and quantiles of one column go to flux-stats' `check_edge_counts` in one call (FLUX ADR 0012; `runner._edge_decisions`), a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | | CI template generator | `fluid_build/forge/core/pipeline_templates.py` — 7-system generator; Jenkins ships the full 11-stage parameterized template with `--install-mode {pypi,dev-source}` | | Forge (AI creation) | `fluid_build/forge/` (templates, generators, extensions) | | Log redaction (global) | `fluid_build/observability/secret_redactor.py` — `SecretRedactingFilter` wired into Python logging | diff --git a/fluid_build/cli/profile_cmd.py b/fluid_build/cli/profile_cmd.py index 44a6e19f..65913621 100644 --- a/fluid_build/cli/profile_cmd.py +++ b/fluid_build/cli/profile_cmd.py @@ -4,9 +4,9 @@ """``fluid profile``: an output-checked profile of one expose, read where its records live. The command reads the expose's records once, inside the environment that holds -them, and writes a FLUX profile document (PROFILE-SPEC format 1-0-0) of -aggregates that passed the disclosure rules. A FLUX Simulation pins the document -by the digest this prints. See :mod:`fluid_build.profiler` for the parts. +them, and writes a FLUX profile document (PROFILE-SPEC format 1-0-0, or 1-1-0 +when it holds a quantile with bins) of aggregates that passed the disclosure +rules. A FLUX Simulation pins the document by the digest this prints. See :mod:`fluid_build.profiler` for the parts. Registered lazily (``cli/bootstrap.py`` ``CORE_COMMANDS``), and only the standard library is imported at module scope, so neither ``fluid --help`` nor ``fluid profile @@ -26,15 +26,14 @@ _DESCRIPTION = """\ Write an output-checked profile of one expose's records, read where they live. -The document follows FLUX PROFILE-SPEC (format 1-0-0) and holds only aggregates -that passed the output checks: no record, no sample, no per-record value. The -checks are ACRO 1.1.0's defaults, named "acro-default" in the document: +The document follows FLUX PROFILE-SPEC (format 1-0-0, or 1-1-0 when it holds a +quantile with bins) and holds only aggregates that passed the output checks: no +record, no sample, no per-record value. The checks are ACRO 1.1.0's defaults, named "acro-default" in the document: - a count below 10 records is withheld, and so is a zero; - every published count is rounded to a multiple of 5; - a sum is withheld when 2 records make up 90% of it, or when what is left after the 2 largest is below 10% of the largest; - - a min, max or quantile is published only when 10 records hold that value, - and a quantile only when at least 10 records lie below it and 10 above it; + - a min, max or quantile is published only when 10 records hold that value; - a mean is published only when the column's count and sum are. A withheld value keeps an SDMX CL_CONF_STATUS code and no value: A (a small count), O or T (dominated by one or two records), M (concentration), D @@ -62,12 +61,21 @@ What the checks leave readable: a grouped statistic lists every group the data forms, published or withheld. A key is always a level the bundle declares (a value outside the declared support is refused), so its presence says only that -the level holds at least one record. A group withheld for its size withholds -every group of its groupBy's histograms and quantiles on its column too, and a -quantile is published only as the one histogram or quantile of its column. A withheld cell of -a histogram is not secondarily suppressed: the rounded counts still bound it, -inside 0 to 9 records, and never above the threshold. A published min or max -shows that no record lies below or above it. A quantile whose value is the +the level holds at least one record. + +The counts a histogram, a level histogram or a quantile publishes at the edges +of one column are checked together, as FLUX ADR 0012 sets, by flux-stats' +check_edge_counts. They sit on one partition (the column's bins, or its +levels). A quantile that declares bins is written as quantile/2, with the bin +that holds its order statistic and the records below that bin and through it, +and the profile then names format 1-1-0; its three cells (below, in and above +the bin) each hold at least 10 records, or none at the ends of the range. A +quantile without bins is published only as the one such statistic of its column. +A group of fewer than 10 records withholds every group of its groupBy. No small +cell, a zero included where zeros are disclosive, can be formed by adding or +subtracting what is published, so more is withheld (D in the summary). Only +then are counts rounded. A published min or +max shows that no record lies below or above it. A quantile whose value is the column's smallest or largest shows the same, and is withheld for it, as a zero is. @@ -86,14 +94,16 @@ # Only aggregates leave: a count under 10 or a zero is withheld, counts are rounded # to 5, a sum 2 records dominate is withheld, and a min, max or quantile is shown only -# when 10 records hold it (ACRO 1.1.0 defaults), a quantile only when at least 10 -# records lie below it and 10 above it, and only as its column's one histogram or -# quantile. A group of fewer than 10 records withholds every group of its groupBy's -# histograms and quantiles on its column. A histogram's or a quantile's withheld values all read C, -# whatever withheld them. A min or max is published though it shows that no record lies -# below or above it, and a quantile at the column's smallest or largest value is withheld -# for showing the same. Output-checking conventions only: no privacy or anonymity -# guarantee. Pin the printed digest in the FLUX Simulation. +# when 10 records hold it (ACRO 1.1.0 defaults). With --bundle, a column's histograms and +# quantiles are checked together (FLUX ADR 0012): on one partition, a quantile with bins +# only with at least 10 records below its bin, in it and above it, and no small cell, +# a zero included, left to be formed by adding or subtracting what is published. A group +# of fewer than 10 records withholds every group of its groupBy. A histogram's or a +# quantile's withheld values all read C, whatever withheld them. A min or max is published +# though it shows that no record lies below or above it, and a quantile at the column's +# smallest or largest value is withheld for showing the same. Output-checking +# conventions only: no privacy or anonymity guarantee. Pin the printed digest in the +# FLUX Simulation. # Each number column also gets mean/1, which FLUX compares a mean property with, on the # grid --fractional-bits names (16 unless given): the fidelity block must declare it. diff --git a/tests/profiler/test_flux_stats_backend.py b/tests/profiler/test_flux_stats_backend.py index cb1950a5..10b6636c 100644 --- a/tests/profiler/test_flux_stats_backend.py +++ b/tests/profiler/test_flux_stats_backend.py @@ -301,4 +301,3 @@ def test_the_backend_reads_a_bundle_through_the_registrys_exact_reader(): (document,) = backend.read_documents("rate: 0.50\nwhen: 2024-01-01\n", name="s", budget=budget) assert document == {"rate": Decimal("0.50"), "when": "2024-01-01"} assert backend.canonicalise(document, name="s") == {"rate": "0.50", "when": "2024-01-01"} - From 3c24e931013634ccd0b48d10ecc7a48eaa752a7c Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:52:21 +0100 Subject: [PATCH 5/9] docs(profile): wrap two long lines of fluid profile's help --- fluid_build/cli/profile_cmd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fluid_build/cli/profile_cmd.py b/fluid_build/cli/profile_cmd.py index 65913621..e72264f5 100644 --- a/fluid_build/cli/profile_cmd.py +++ b/fluid_build/cli/profile_cmd.py @@ -6,7 +6,8 @@ The command reads the expose's records once, inside the environment that holds them, and writes a FLUX profile document (PROFILE-SPEC format 1-0-0, or 1-1-0 when it holds a quantile with bins) of aggregates that passed the disclosure -rules. A FLUX Simulation pins the document by the digest this prints. See :mod:`fluid_build.profiler` for the parts. +rules. A FLUX Simulation pins the document by the digest this prints. See +:mod:`fluid_build.profiler` for the parts. Registered lazily (``cli/bootstrap.py`` ``CORE_COMMANDS``), and only the standard library is imported at module scope, so neither ``fluid --help`` nor ``fluid profile @@ -28,7 +29,8 @@ The document follows FLUX PROFILE-SPEC (format 1-0-0, or 1-1-0 when it holds a quantile with bins) and holds only aggregates that passed the output checks: no -record, no sample, no per-record value. The checks are ACRO 1.1.0's defaults, named "acro-default" in the document: +record, no sample, no per-record value. The checks are ACRO 1.1.0's defaults, named +"acro-default" in the document: - a count below 10 records is withheld, and so is a zero; - every published count is rounded to a multiple of 5; - a sum is withheld when 2 records make up 90% of it, or when what is left From 12e7b10c150f7a286edfca29190abdcf3a2d141b Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:03:44 +0100 Subject: [PATCH 6/9] docs(profile): name flux-stats 0.7.0 as the first release the profiler can use flux-stats 0.7.0 is on PyPI, so the pin's comment and the install message's note name it as the first release that holds edge_counts, exact_yaml and quantile/2, instead of a placeholder for the maintainers to fill in. --- fluid_build/profiler/_stats.py | 4 ++-- pyproject.toml | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fluid_build/profiler/_stats.py b/fluid_build/profiler/_stats.py index 1b6c65ed..d7919f11 100644 --- a/fluid_build/profiler/_stats.py +++ b/fluid_build/profiler/_stats.py @@ -856,8 +856,8 @@ def load_backend() -> StatisticsBackend: #: The flux-stats range ``[profile]`` names in ``pyproject.toml``. The CLI cannot read #: that file once installed, so the install message repeats it; #: ``tests/profiler/test_flux_stats_pin.py`` keeps the two equal. -#: TODO: the founder names the real version when flux-stats releases it; 0.7.0 is the -#: first release that holds ``edge_counts``, ``exact_yaml`` and ``quantile/2``. +#: 0.7.0 is the first flux-stats release that holds ``edge_counts``, ``exact_yaml`` and +#: ``quantile/2``. FLUX_STATS_REQUIREMENT = "flux-stats[yaml]>=0.7.0,<0.8" #: The extra carries the range, and pip replaces a copy outside it (the 0.0.1 #: placeholder, or a release newer than this module knows). diff --git a/pyproject.toml b/pyproject.toml index b65b1450..a5b73af3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -413,8 +413,7 @@ lineage = [ # fluid_build/profiler/_stats.py imports, so the range stops at the next minor. # That module repeats the requirement in its install message, and # tests/profiler/test_flux_stats_pin.py keeps the two equal. -# TODO: the founder names the real flux-stats version when it is released; 0.7.0 is -# the first release that holds these modules. +# 0.7.0 is the first flux-stats release that holds these modules. profile = [ "duckdb>=1.5.0", "flux-stats[yaml]>=0.7.0,<0.8", From 1a3efc884486d04a0ee6818844aafa25bea0da7a Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 09:40:12 +0100 Subject: [PATCH 7/9] refactor(profile): check a declared statistic's groups in one of two helpers A histogram's or a quantile's groups follow the output check on its column, and a mean's or a support's their own checks with their siblings. Each path is now a helper of its own, so the entry that writes a statistic only picks one. --- fluid_build/profiler/runner.py | 66 +++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/fluid_build/profiler/runner.py b/fluid_build/profiler/runner.py index 1340173d..7551937f 100644 --- a/fluid_build/profiler/runner.py +++ b/fluid_build/profiler/runner.py @@ -809,29 +809,11 @@ def _declared_entry( its column, and a mean's and a support's from their own checks. *tallies* count the groups and the histogram cells that are withheld, by the check. """ - tally, cell_tally = tallies groups = measured.groups - written: List[Dict[str, Any]] = [] if decision is not None: - for group in groups: - if decision.group_status[group.key] == disclosure.FREE: - tally[disclosure.FREE] += 1 - published = _published_edge_group(statistic, group, decision, rule, cell_tally) - written.append({**published, "status": disclosure.FREE}) - else: - tally[_code(decision.group_reason[group.key])] += 1 - written.append({"group": group.key, "status": disclosure.WITHHELD}) + written = _edge_checked_groups(statistic, measured, rule, tallies, decision) else: - checked = [_check_group(statistic, group, rule, fractional_bits) for group in groups] - verdicts = [verdict for verdict, _ in checked] - if statistic.group_by is not None: - verdicts = _suppressed(statistic, verdicts, measured, rule, fractional_bits) - for group, verdict, (_, published) in zip(groups, verdicts, checked, strict=True): - tally[verdict.status] += 1 - if verdict.published: - written.append({**published, "status": disclosure.FREE}) - else: - written.append({"group": group.key, "status": verdict.status}) + written = _own_checked_groups(statistic, measured, rule, tallies[0], fractional_bits) entry: Dict[str, Any] = { "definition": statistic.identifier, "sampleSize": disclosure.sample_size(sum(group.k for group in groups), rule), @@ -844,6 +826,50 @@ def _declared_entry( return entry +def _edge_checked_groups( + statistic: DeclaredStatistic, + measured: MeasuredStatistic, + rule: disclosure.DisclosureRule, + tallies: Tuple["Counter[str]", "Counter[str]"], + decision: EdgeDecision, +) -> List[Dict[str, Any]]: + """A histogram's or a quantile's groups, as the output check on its column decided them.""" + tally, cell_tally = tallies + written: List[Dict[str, Any]] = [] + for group in measured.groups: + if decision.group_status[group.key] == disclosure.FREE: + tally[disclosure.FREE] += 1 + published = _published_edge_group(statistic, group, decision, rule, cell_tally) + written.append({**published, "status": disclosure.FREE}) + else: + tally[_code(decision.group_reason[group.key])] += 1 + written.append({"group": group.key, "status": disclosure.WITHHELD}) + return written + + +def _own_checked_groups( + statistic: DeclaredStatistic, + measured: MeasuredStatistic, + rule: disclosure.DisclosureRule, + tally: "Counter[str]", + fractional_bits: int, +) -> List[Dict[str, Any]]: + """A mean's or a support's groups, each by its own check, siblings withheld with a small one.""" + groups = measured.groups + checked = [_check_group(statistic, group, rule, fractional_bits) for group in groups] + verdicts = [verdict for verdict, _ in checked] + if statistic.group_by is not None: + verdicts = _suppressed(statistic, verdicts, measured, rule, fractional_bits) + written: List[Dict[str, Any]] = [] + for group, verdict, (_, published) in zip(groups, verdicts, checked, strict=True): + tally[verdict.status] += 1 + if verdict.published: + written.append({**published, "status": disclosure.FREE}) + else: + written.append({"group": group.key, "status": verdict.status}) + return written + + def _published_edge_group( statistic: DeclaredStatistic, group: MeasuredGroup, From 82675aa7c7c87abaa9646e1a22e36f416cd3f466 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:02:03 +0100 Subject: [PATCH 8/9] fix(profile): refuse bins off the grid, withhold a group with no cell left, say only a refusal's code From the reviews of this branch: - Bins whose edges are not strictly increasing grid points (0.4 and 0.6 at fractionalBits 0) left the edge check with no partition, and flux-stats raised a KeyError the profile did not catch. Such a block is now refused before a record is read, by the registry's own rule (FLX-5004), as the fidelity check refuses it. - A histogram group the check let through with every cell withheld was written as published, with null cells, and the report named the statistic as answered. It is withheld whole again, as a group whose every cell is withheld was before the check ran on edges. - fluid profile --help said no small cell can be formed from what is published. It now says the check covers one column's histograms, level histograms and quantiles, and names what it does not see. - A refusal from inside check_edge_counts names its code, not the registry's words, which may give counts, and FLX-1007 says its title, not the colliding key. --- fluid_build/cli/profile_cmd.py | 13 ++-- fluid_build/profiler/_stats.py | 51 +++++++++++--- fluid_build/profiler/runner.py | 33 ++++++++- tests/profiler/test_bundle_flux_stats.py | 85 +++++++++++++++++++++-- tests/profiler/test_flux_stats_backend.py | 14 ++++ 5 files changed, 176 insertions(+), 20 deletions(-) diff --git a/fluid_build/cli/profile_cmd.py b/fluid_build/cli/profile_cmd.py index e72264f5..06dac030 100644 --- a/fluid_build/cli/profile_cmd.py +++ b/fluid_build/cli/profile_cmd.py @@ -73,10 +73,15 @@ and the profile then names format 1-1-0; its three cells (below, in and above the bin) each hold at least 10 records, or none at the ends of the range. A quantile without bins is published only as the one such statistic of its column. -A group of fewer than 10 records withholds every group of its groupBy. No small -cell, a zero included where zeros are disclosive, can be formed by adding or -subtracting what is published, so more is withheld (D in the summary). Only -then are counts rounded. A published min or +A group of fewer than 10 records withholds every group of its groupBy, and a +group whose every cell is withheld is withheld whole. Across those statistics +of one column, no small cell, a zero included where zeros are disclosive, can be +formed by adding or subtracting what they publish, so more is withheld (D in +the summary). Only then are counts rounded. The check does not see a column's +support, which lists the levels that hold records, or the group sizes of a +statistic grouped by the column, which repeat its level counts; and a declared +level that holds no record is absent from a grouped statistic, which shows its +zero. A published min or max shows that no record lies below or above it. A quantile whose value is the column's smallest or largest shows the same, and is withheld for it, as a zero is. diff --git a/fluid_build/profiler/_stats.py b/fluid_build/profiler/_stats.py index d7919f11..f46b78a1 100644 --- a/fluid_build/profiler/_stats.py +++ b/fluid_build/profiler/_stats.py @@ -91,6 +91,10 @@ class DocumentRefused(ValueError): """ +#: The canonical form's refusal of two source keys that read as one key. +_KEY_COLLISION = "FLX-1007" + + class BlockRefused(ValueError): """The registry's parser refused the fidelity block a Simulation declares. @@ -562,7 +566,13 @@ def canonicalise(self, value: Any, *, name: str) -> Any: try: return self._canonical.canonicalise(value) except self._refused as error: - detail = error.detail or error.code.title + # FLX-1007 names the key two source keys read as, and a bundle key may + # hold anything, so only the code's title is said for it. + detail = ( + error.code.title + if error.code.code == _KEY_COLLISION + else (error.detail or error.code.title) + ) raise BundleTextRefused(error.code.code, f"{name}: {detail}") from None def check_document(self, text: str) -> None: @@ -582,6 +592,7 @@ def _fidelity(self) -> Any: # By name, so a flux-stats that lacks the module raises here. fidelity = import_module("flux_stats.fidelity") reading = import_module("flux_stats.reading") + fixedpoint = import_module("flux_stats.fixedpoint") from flux_stats.registry import definition as definition_named from flux_stats.spelling import render_exact except ImportError as error: @@ -593,6 +604,7 @@ def _fidelity(self) -> Any: self._grammar = SimpleNamespace( fidelity=fidelity, reading=reading, + fixedpoint=fixedpoint, definition_named=definition_named, render_exact=render_exact, ) @@ -615,8 +627,10 @@ def read_block( statistics: List[DeclaredStatistic] = [] left_out: List[Tuple[str, str]] = [] seen = set() - for declaration in parsed.properties: + for index, declaration in enumerate(parsed.properties): kind = declaration.kind.value + if declaration.bins: + self._refuse_off_grid_bins(grammar, parsed, index, declaration.bins) if kind == "correlation": first, second = declaration.columns left_out.append((f"correlation of {first!r} and {second!r}", _CORRELATION_LEFT_OUT)) @@ -638,6 +652,21 @@ def read_block( _block=parsed, ) + def _refuse_off_grid_bins(self, grammar: Any, parsed: Any, index: int, bins: Any) -> None: + """Refuse bins the fidelity check could not compare, before a record is read. + + The check bins a record exactly only where every edge is a grid point and the + edges strictly increase, and refuses any other (``FLX-5004``), so the profile + applies the registry's own rule. Edges that fall to one grid point would also + leave the output check over a column with no partition to read. The message + names the property and the edge's place, never a value. + """ + try: + grammar.fixedpoint.epsilon_marginal_histogram(parsed.grid, bins) + except self._refused as error: + detail = error.detail or error.code.title + raise BlockRefused(error.code.code, f"properties[{index}]: {detail}") from None + @staticmethod def _declared(grammar: Any, parsed: Any, declaration: Any) -> DeclaredStatistic: """One property of the parsed block as the statistic a profile carries for it.""" @@ -790,13 +819,17 @@ def check_edge_counts( cell_status=dict(each.cell_status), ) ) - decisions = edges.check_edge_counts( - asked, - total=total, - minimum_cell=minimum_cell, - zeros_disclosive=zeros_disclosive, - round_count=round_count, - ) + try: + decisions = edges.check_edge_counts( + asked, + total=total, + minimum_cell=minimum_cell, + zeros_disclosive=zeros_disclosive, + round_count=round_count, + ) + except self._refused as error: + # The registry's words may give counts; its code alone says what it refused. + raise ValueError(error.code.code) from None return tuple( self._decided(grammar, each, decision) for each, decision in zip(statistics, decisions, strict=True) diff --git a/fluid_build/profiler/runner.py b/fluid_build/profiler/runner.py index 7551937f..91d7aed6 100644 --- a/fluid_build/profiler/runner.py +++ b/fluid_build/profiler/runner.py @@ -837,16 +837,45 @@ def _edge_checked_groups( tally, cell_tally = tallies written: List[Dict[str, Any]] = [] for group in measured.groups: - if decision.group_status[group.key] == disclosure.FREE: + if decision.group_status[group.key] == disclosure.FREE and not _every_cell_withheld( + statistic, decision, group.key + ): tally[disclosure.FREE] += 1 published = _published_edge_group(statistic, group, decision, rule, cell_tally) written.append({**published, "status": disclosure.FREE}) else: - tally[_code(decision.group_reason[group.key])] += 1 + tally[_withheld_code(statistic, decision, group.key)] += 1 written.append({"group": group.key, "status": disclosure.WITHHELD}) return written +def _every_cell_withheld(statistic: DeclaredStatistic, decision: EdgeDecision, key: str) -> bool: + """Whether a histogram group the check lets through holds no published cell. + + Such a group says nothing but its size, so it is withheld whole, as a group whose + every cell is withheld always was. A quantile's group has no cells. + """ + if statistic.kind == "quantile": + return False + return all(status != disclosure.FREE for status in decision.cell_status[key]) + + +def _withheld_code(statistic: DeclaredStatistic, decision: EdgeDecision, key: str) -> str: + """The code a withheld edge group is tallied under: its own reason, else its cells'. + + A group withheld whole because every cell is withheld counts as small (``A``) where + one of its cells was withheld for its own size, and as secondary (``D``) otherwise. + """ + if decision.group_status[key] != disclosure.FREE: + return _code(decision.group_reason[key]) + codes = { + _code(reason) + for reason, status in zip(decision.cell_reason[key], decision.cell_status[key], strict=True) + if status != disclosure.FREE + } + return disclosure.SMALL_COUNT if disclosure.SMALL_COUNT in codes else disclosure.SECONDARY + + def _own_checked_groups( statistic: DeclaredStatistic, measured: MeasuredStatistic, diff --git a/tests/profiler/test_bundle_flux_stats.py b/tests/profiler/test_bundle_flux_stats.py index c9d5ef41..ac44fb4b 100644 --- a/tests/profiler/test_bundle_flux_stats.py +++ b/tests/profiler/test_bundle_flux_stats.py @@ -447,7 +447,9 @@ def test_a_histogram_and_a_coarser_one_of_a_column_give_no_withheld_cell_back(tm coarse = _histogram(["0", "50", "100"]) result = _profile(tmp_path, rows, [fine, coarse]) fine_group, coarse_group = (histogram["groups"][0] for histogram in _histograms(result)) - fine_cells, coarse_cells = fine_group["binCounts"], coarse_group["binCounts"] + # A group whose every cell is withheld is withheld whole, with no cells written. + fine_cells = fine_group.get("binCounts", [None] * 4) + coarse_cells = coarse_group.get("binCounts", [None] * 2) assert fine_cells[1] is None # the 5 itself for cell, inside in ((0, fine_cells[:2]), (1, fine_cells[2:])): if coarse_cells[cell] is not None: @@ -605,6 +607,8 @@ def test_a_quantile_beside_a_histogram_of_another_column_is_published(tmp_path): def test_a_small_group_of_one_column_leaves_another_columns_groups_published(tmp_path): rows = _rows() + for index, row in enumerate(rows): + row["qty"] = "60" if index % 2 else "10" # each group of qty fills both bins for index, row in enumerate(rows[:15]): row["market"] = "DE" if index < 12: @@ -617,7 +621,10 @@ def test_a_small_group_of_one_column_leaves_another_columns_groups_published(tmp amount = _entry(result, "amount", "marginalHistogram/1", "market") assert {g["status"] for g in amount["groups"]} == {"C"} qty = _entry(result, "qty", "marginalHistogram/1", "market") - assert {g["status"] for g in qty["groups"]} == {"F"} + statuses = {g["group"]: g["status"] for g in qty["groups"]} + # DE's 15 records split 8 and 7, so both its cells are small and it is withheld whole; + # the other markets' groups of qty are published, whatever amount's small group did. + assert statuses == {"DE": "C", "FR": "F", "GB": "F", "IE": "F"} def test_rows_no_group_holds_that_the_block_refuses_withhold_a_grouped_histogram(tmp_path): @@ -721,9 +728,11 @@ def test_a_zero_cell_is_withheld_with_the_cell_that_gives_it_back(tmp_path): ] result = _profile(tmp_path, rows, properties) (group,) = _entry(result, "amount", "marginalHistogram/1")["groups"] - # The zero cell is withheld, and the 120 beside it would give it back: it goes too. - assert group["binCounts"] == [None, None] and group["cellStatus"] == ["C", "C"] - assert result.withheld_cells == {"A": 1, "D": 1} + # The zero cell is withheld, and the 120 beside it would give it back: it goes too, + # and a group with no cell left says nothing, so it is withheld whole, as small. + assert group == {"group": "", "status": "C"} + assert result.withheld_cells == {} # no cell of it is written + assert result.withheld["A"] >= 1 # the group, among the profile's withheld values def test_a_small_table_withholds_the_histogram_whole(tmp_path): @@ -1112,3 +1121,69 @@ def test_a_report_with_no_withheld_histogram_value_says_nothing_of_c(tmp_path, m out = " ".join(capsys.readouterr().out.split()) assert rc == 0, out assert _ONE_CODE not in out and "Withheld histogram cells" not in out + + +@pytest.mark.parametrize( + "declared", + [ + { + "property": "marginalHistogram", + "bins": ["0", "0.4", "0.6", "100"], + "tolerance": _DISTANCE, + }, + {"property": "quantile", "level": "0.5", "bins": ["0", "0.4", "0.6", "100"]}, + ], + ids=["histogram", "quantile"], +) +def test_bins_that_are_no_grid_points_are_refused_before_any_record_is_read(tmp_path, declared): + # At fractionalBits 0 the edges 0.4 and 0.6 are no grid points, and both fall to the + # grid point 0: the fidelity check could not bin a record on them (FLX-5004), and the + # output check would have no partition to read. The block is refused as the check + # refuses it, naming the property and the edge, never a value. + prop = {"column": "amount", "tolerance": _ABSOLUTE, **declared} + directory = _bundle(tmp_path, [prop], bits=0) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) + with pytest.raises(ProfileError) as refused: + profile_expose( + CONTRACT, + EXPOSE, + source=str(tmp_path / "missing.ndjson"), # never read + backend=_stats.load_backend(), + cwd=tmp_path, + bundle=block, + ) + assert refused.value.event == "profile_bundle_block_refused" + said = str(refused.value) + assert "FLX-5004" in said and "properties[0]" in said and "bins[1]" in said + assert "0.4" not in said + + +def test_a_histogram_group_with_no_cell_left_is_withheld_whole(tmp_path): + # DE's 12 records fall 8 and 4 in the two bins, so both its cells are small. A group + # with no cell left says nothing but its size, so it is withheld whole, as a group + # whose every cell is withheld always was, and the histogram is named as withheld in + # part, not as answered. + histogram = _histogram(["0", "50", "100"], groupBy="market") + result = _profile(tmp_path, _partly_rows(), [histogram]) + entry = _entry(result, "amount", "marginalHistogram/1", "market") + groups = {group["group"]: group for group in entry["groups"]} + assert groups["DE"] == {"group": "DE", "status": "C"} + assert {key for key, group in groups.items() if group["status"] == "F"} == {"FR", "GB", "IE"} + (named,) = [one for one in result.partly_withheld_statistics if one[1] == "marginalHistogram/1"] + assert named[-2:] == (3, 4) + load_profile_text(result.canonical.decode("utf-8")) + + +def test_a_refusal_inside_the_output_check_names_only_its_code(tmp_path, monkeypatch): + # The registry's words for a refusal may give counts, so the profile says its code. + from flux_stats import edge_counts, fixedpoint + from flux_stats.errors import FluxError + + def refuse(*_args: Any, **_kwargs: Any) -> Any: + raise FluxError(fixedpoint.FLX_5004, "witness ranks 3/7 are not a sub-count of 9") + + monkeypatch.setattr(edge_counts, "check_edge_counts", refuse) + with pytest.raises(ProfileError) as refused: + _profile(tmp_path, _rows(), [_histogram(["0", "50", "100"])]) + said = str(refused.value) + assert "FLX-5004" in said and "3/7" not in said and "9" not in said.split("FLX-5004")[-1] diff --git a/tests/profiler/test_flux_stats_backend.py b/tests/profiler/test_flux_stats_backend.py index 10b6636c..030b11bc 100644 --- a/tests/profiler/test_flux_stats_backend.py +++ b/tests/profiler/test_flux_stats_backend.py @@ -28,6 +28,7 @@ import pytest from fluid_build.profiler._stats import ( + BundleTextRefused, DocumentRefused, ReadRefused, StatisticsBackend, @@ -301,3 +302,16 @@ def test_the_backend_reads_a_bundle_through_the_registrys_exact_reader(): (document,) = backend.read_documents("rate: 0.50\nwhen: 2024-01-01\n", name="s", budget=budget) assert document == {"rate": Decimal("0.50"), "when": "2024-01-01"} assert backend.canonicalise(document, name="s") == {"rate": "0.50", "when": "2024-01-01"} + + +def test_two_keys_that_read_as_one_are_refused_without_naming_the_key(): + # A bundle key may hold anything, so the refusal of two source keys that read as one + # canonical key (FLX-1007) says the code's title, not the key. + backend = load_backend() + composed, decomposed = chr(0xE9), "e" + chr(0x301) # one key once normalised + with pytest.raises(BundleTextRefused) as refused: + backend.canonicalise( + {f"{composed}-hunter2": 1, f"{decomposed}-hunter2": 2}, name="simulation.flux.yml" + ) + said = str(refused.value) + assert "FLX-1007" in said and "simulation.flux.yml" in said and "hunter2" not in said From dbf86c70dd0fc5056ac17daa51c7146c4859797d Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:04:45 +0100 Subject: [PATCH 9/9] docs(changelog): fluid profile checks a column's edge counts with flux-stats 0.7.0 Under the Unreleased section after 0.34.0: quantile/2 and format 1-1-0 (Added), the bundle read through flux-stats' exact reader, the FLX-5004 refusal of bins off the grid and the flux-stats 0.7.0 pin (Changed), and the joint check of a column's histograms, level histograms and quantiles, with what it does not see (Security). The released sections, #995's note in 0.33.0 included, are untouched. --- CHANGELOG.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e306fcb5..ce4fd8b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`fluid profile --bundle` publishes a quantile with bins as its bin.** A `quantile` property + that declares `bins` is now answered with `quantile/2`: the group states the declared bin that + holds the order statistic at its level, and the records below that bin and through it, and no + witness. A reader checks that the rounded counts still put the order statistic in that bin, + and a group whose rounded counts do not is withheld. A profile that holds one names format + `1-1-0`; every other profile keeps `1-0-0` and its bytes. A quantile with bins is published + beside a histogram that shares its bins, where a `quantile/1` is withheld. + +### Changed + +- **`fluid profile --bundle` reads a bundle with flux-stats.** The bundle's YAML is read by + flux-stats' exact reader and its block put in the canonical form by flux-stats, instead of a + copy of the engine's code in forge-cli. The bounds hold as before (aliases, merge keys, digits + written out, nesting depth, document size, one budget across a bundle's files), and a refusal + still names the file, and the line where the reader reports one, never a value. Its words + differ in places: a YAML error names the line and column where the reader found it, the most + values a document may write out through its aliases is now 1,000,000, and the characters it + writes out are bounded too. A block whose bins are not strictly increasing grid points is + refused before any record is read (`FLX-5004`), as the fidelity check refuses it. The + `[profile]` extra needs `flux-stats[yaml]>=0.7.0,<0.8`: 0.7.0 is the first flux-stats release + that holds these readers and the check below. + +### Security + +- **A histogram, a level histogram and a quantile of one column are checked together (FLUX ADR + 0012).** 0.33.0's group-size, quantile and sibling checks were each one part of a rule, and a + reader could still subtract one histogram's published counts from another's, or from a level + histogram's group size, and rebuild a withheld cell, a zero included. `fluid profile` now + passes every histogram, level histogram and quantile of a column to flux-stats' + `check_edge_counts` in one call, and publishes what it lets through. More is withheld: + - differencing between histograms: a histogram and a coarser one of one column, or a published + cell and the column's count, no longer give a withheld cell back by subtraction, and a cell + that does goes with it, counted as `D`; + - a level histogram is checked over its levels as it is over bins, so one small level + withholds a neighbour; + - a zero cell a reader can form, where zeros are disclosive: cells of 12, 0 and 12 records + withhold one of the 12s with the zero; + - a histogram that does not sit on the column's one partition of bins, and a group whose + rounded counts no longer prove its quantile; + - a group that is withheld whole beside rows no group holds that the block cannot read. + + The summary counts a cell or group withheld for its size or rounding as `A`, and one withheld + so that another cannot be recovered as `D`. A group whose every cell is withheld is still + withheld whole. The check does not see a column's `support/1`, which lists the levels that hold + records, the group sizes of a statistic grouped by the column, which repeat its level counts, + or a declared level that holds no record, which is absent from a grouped statistic; `fluid + profile --help` says so. + ## [0.34.0] — 2026-11-19 ### Added