diff --git a/AGENTS.md b/AGENTS.md index 25a0f05c..bec70145 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,7 +164,7 @@ forge-cli/ | Bundle builder | `fluid_build/forge/core/bundle.py` — deterministic tgz + MANIFEST + fragment extraction + `$source` sentinel | | Build runners | `fluid_build/build_runners/{dbt,python}/` — extracted from legacy `cli/execute.py` in Phase 1 | | Masking at landing | `fluid_build/build_runners/_masking.py` — the DuckDB runner applies `exposes[].policy.privacy.masking[]` inside the COPY (file, S3, BigQuery staging, DLQ); secrets only from env (`FLUID_PII_HASH_SECRET`, `FLUID_PII_TOKENIZATION_KEY`, `FLUID_PII_ENCRYPTION_SECRET_KEY`, or `params.saltEnv`/`keyEnv`), held in UDF closures, never in SQL; an unappliable rule refuses the build, and so does a DuckDB without numpy (its `create_function` needs it; `[local]` names numpy, and the refusal names the extra). `cli/_verify_masking.py` is stage 9's shape check (local DuckDB, Athena in the count query) | -| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads one expose's records where they live (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC 1-0-0 document (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). With `--bundle DIR` it also answers the fidelity block of DIR's one Simulation: `bundle.py` reads `*.flux.yml` as the engine does (its exact YAML loader, then KERNEL-SPEC 2.1 canonical form), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | +| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads one expose's records where they live (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC 1-0-0 document (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). With `--bundle DIR` it also answers the fidelity block of DIR's one Simulation: `bundle.py` reads `*.flux.yml` as the engine does (flux-stats' `flux_stats.exact_yaml` and `flux_stats.canonical`, through the seam, one `ReadBudget` per bundle), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a quantile with bins is `quantile/2` (the profile then names format 1-1-0), the histograms and quantiles of one column go to flux-stats' `check_edge_counts` in one call (FLUX ADR 0012; `runner._edge_decisions`), a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | | CI template generator | `fluid_build/forge/core/pipeline_templates.py` — 7-system generator; Jenkins ships the full 11-stage parameterized template with `--install-mode {pypi,dev-source}` | | Forge (AI creation) | `fluid_build/forge/` (templates, generators, extensions) | | Log redaction (global) | `fluid_build/observability/secret_redactor.py` — `SecretRedactingFilter` wired into Python logging | diff --git a/CHANGELOG.md b/CHANGELOG.md index e306fcb5..ce4fd8b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`fluid profile --bundle` publishes a quantile with bins as its bin.** A `quantile` property + that declares `bins` is now answered with `quantile/2`: the group states the declared bin that + holds the order statistic at its level, and the records below that bin and through it, and no + witness. A reader checks that the rounded counts still put the order statistic in that bin, + and a group whose rounded counts do not is withheld. A profile that holds one names format + `1-1-0`; every other profile keeps `1-0-0` and its bytes. A quantile with bins is published + beside a histogram that shares its bins, where a `quantile/1` is withheld. + +### Changed + +- **`fluid profile --bundle` reads a bundle with flux-stats.** The bundle's YAML is read by + flux-stats' exact reader and its block put in the canonical form by flux-stats, instead of a + copy of the engine's code in forge-cli. The bounds hold as before (aliases, merge keys, digits + written out, nesting depth, document size, one budget across a bundle's files), and a refusal + still names the file, and the line where the reader reports one, never a value. Its words + differ in places: a YAML error names the line and column where the reader found it, the most + values a document may write out through its aliases is now 1,000,000, and the characters it + writes out are bounded too. A block whose bins are not strictly increasing grid points is + refused before any record is read (`FLX-5004`), as the fidelity check refuses it. The + `[profile]` extra needs `flux-stats[yaml]>=0.7.0,<0.8`: 0.7.0 is the first flux-stats release + that holds these readers and the check below. + +### Security + +- **A histogram, a level histogram and a quantile of one column are checked together (FLUX ADR + 0012).** 0.33.0's group-size, quantile and sibling checks were each one part of a rule, and a + reader could still subtract one histogram's published counts from another's, or from a level + histogram's group size, and rebuild a withheld cell, a zero included. `fluid profile` now + passes every histogram, level histogram and quantile of a column to flux-stats' + `check_edge_counts` in one call, and publishes what it lets through. More is withheld: + - differencing between histograms: a histogram and a coarser one of one column, or a published + cell and the column's count, no longer give a withheld cell back by subtraction, and a cell + that does goes with it, counted as `D`; + - a level histogram is checked over its levels as it is over bins, so one small level + withholds a neighbour; + - a zero cell a reader can form, where zeros are disclosive: cells of 12, 0 and 12 records + withhold one of the 12s with the zero; + - a histogram that does not sit on the column's one partition of bins, and a group whose + rounded counts no longer prove its quantile; + - a group that is withheld whole beside rows no group holds that the block cannot read. + + The summary counts a cell or group withheld for its size or rounding as `A`, and one withheld + so that another cannot be recovered as `D`. A group whose every cell is withheld is still + withheld whole. The check does not see a column's `support/1`, which lists the levels that hold + records, the group sizes of a statistic grouped by the column, which repeat its level counts, + or a declared level that holds no record, which is absent from a grouped statistic; `fluid + profile --help` says so. + ## [0.34.0] — 2026-11-19 ### Added diff --git a/fluid_build/cli/profile_cmd.py b/fluid_build/cli/profile_cmd.py index 69758b28..06dac030 100644 --- a/fluid_build/cli/profile_cmd.py +++ b/fluid_build/cli/profile_cmd.py @@ -4,9 +4,10 @@ """``fluid profile``: an output-checked profile of one expose, read where its records live. The command reads the expose's records once, inside the environment that holds -them, and writes a FLUX profile document (PROFILE-SPEC format 1-0-0) of -aggregates that passed the disclosure rules. A FLUX Simulation pins the document -by the digest this prints. See :mod:`fluid_build.profiler` for the parts. +them, and writes a FLUX profile document (PROFILE-SPEC format 1-0-0, or 1-1-0 +when it holds a quantile with bins) of aggregates that passed the disclosure +rules. A FLUX Simulation pins the document by the digest this prints. See +:mod:`fluid_build.profiler` for the parts. Registered lazily (``cli/bootstrap.py`` ``CORE_COMMANDS``), and only the standard library is imported at module scope, so neither ``fluid --help`` nor ``fluid profile @@ -26,15 +27,15 @@ _DESCRIPTION = """\ Write an output-checked profile of one expose's records, read where they live. -The document follows FLUX PROFILE-SPEC (format 1-0-0) and holds only aggregates -that passed the output checks: no record, no sample, no per-record value. The -checks are ACRO 1.1.0's defaults, named "acro-default" in the document: +The document follows FLUX PROFILE-SPEC (format 1-0-0, or 1-1-0 when it holds a +quantile with bins) and holds only aggregates that passed the output checks: no +record, no sample, no per-record value. The checks are ACRO 1.1.0's defaults, named +"acro-default" in the document: - a count below 10 records is withheld, and so is a zero; - every published count is rounded to a multiple of 5; - a sum is withheld when 2 records make up 90% of it, or when what is left after the 2 largest is below 10% of the largest; - - a min, max or quantile is published only when 10 records hold that value, - and a quantile only when at least 10 records lie below it and 10 above it; + - a min, max or quantile is published only when 10 records hold that value; - a mean is published only when the column's count and sum are. A withheld value keeps an SDMX CL_CONF_STATUS code and no value: A (a small count), O or T (dominated by one or two records), M (concentration), D @@ -62,12 +63,26 @@ What the checks leave readable: a grouped statistic lists every group the data forms, published or withheld. A key is always a level the bundle declares (a value outside the declared support is refused), so its presence says only that -the level holds at least one record. A group withheld for its size withholds -every group of its groupBy's histograms and quantiles on its column too, and a -quantile is published only as the one histogram or quantile of its column. A withheld cell of -a histogram is not secondarily suppressed: the rounded counts still bound it, -inside 0 to 9 records, and never above the threshold. A published min or max -shows that no record lies below or above it. A quantile whose value is the +the level holds at least one record. + +The counts a histogram, a level histogram or a quantile publishes at the edges +of one column are checked together, as FLUX ADR 0012 sets, by flux-stats' +check_edge_counts. They sit on one partition (the column's bins, or its +levels). A quantile that declares bins is written as quantile/2, with the bin +that holds its order statistic and the records below that bin and through it, +and the profile then names format 1-1-0; its three cells (below, in and above +the bin) each hold at least 10 records, or none at the ends of the range. A +quantile without bins is published only as the one such statistic of its column. +A group of fewer than 10 records withholds every group of its groupBy, and a +group whose every cell is withheld is withheld whole. Across those statistics +of one column, no small cell, a zero included where zeros are disclosive, can be +formed by adding or subtracting what they publish, so more is withheld (D in +the summary). Only then are counts rounded. The check does not see a column's +support, which lists the levels that hold records, or the group sizes of a +statistic grouped by the column, which repeat its level counts; and a declared +level that holds no record is absent from a grouped statistic, which shows its +zero. A published min or +max shows that no record lies below or above it. A quantile whose value is the column's smallest or largest shows the same, and is withheld for it, as a zero is. @@ -86,14 +101,16 @@ # Only aggregates leave: a count under 10 or a zero is withheld, counts are rounded # to 5, a sum 2 records dominate is withheld, and a min, max or quantile is shown only -# when 10 records hold it (ACRO 1.1.0 defaults), a quantile only when at least 10 -# records lie below it and 10 above it, and only as its column's one histogram or -# quantile. A group of fewer than 10 records withholds every group of its groupBy's -# histograms and quantiles on its column. A histogram's or a quantile's withheld values all read C, -# whatever withheld them. A min or max is published though it shows that no record lies -# below or above it, and a quantile at the column's smallest or largest value is withheld -# for showing the same. Output-checking conventions only: no privacy or anonymity -# guarantee. Pin the printed digest in the FLUX Simulation. +# when 10 records hold it (ACRO 1.1.0 defaults). With --bundle, a column's histograms and +# quantiles are checked together (FLUX ADR 0012): on one partition, a quantile with bins +# only with at least 10 records below its bin, in it and above it, and no small cell, +# a zero included, left to be formed by adding or subtracting what is published. A group +# of fewer than 10 records withholds every group of its groupBy. A histogram's or a +# quantile's withheld values all read C, whatever withheld them. A min or max is published +# though it shows that no record lies below or above it, and a quantile at the column's +# smallest or largest value is withheld for showing the same. Output-checking +# conventions only: no privacy or anonymity guarantee. Pin the printed digest in the +# FLUX Simulation. # Each number column also gets mean/1, which FLUX compares a mean property with, on the # grid --fractional-bits names (16 unless given): the fidelity block must declare it. @@ -216,7 +233,7 @@ def run(args: argparse.Namespace, logger: logging.Logger) -> int: try: expose_id = runner.resolve_expose_id(contract, args.expose_id) out = _out_path(args.out, expose_id, args.source) - block = bundle.read_bundle_block(bundle_dir, expose_id) if bundle_dir else None + block = bundle.read_bundle_block(bundle_dir, expose_id, backend) if bundle_dir else None result = runner.profile_expose( contract, expose_id, diff --git a/fluid_build/profiler/_stats.py b/fluid_build/profiler/_stats.py index d6475b95..f46b78a1 100644 --- a/fluid_build/profiler/_stats.py +++ b/fluid_build/profiler/_stats.py @@ -52,9 +52,12 @@ __all__ = [ "BlockRefused", + "BundleTextRefused", "DeclaredBlock", "DeclaredStatistic", "DocumentRefused", + "EdgeDecision", + "EdgeStatistic", "MeasureRefused", "MeasuredGroup", "MeasuredStatistic", @@ -63,6 +66,7 @@ "StatisticDefinition", "StatisticsBackend", "StatisticsUnavailable", + "WITHHELD_ROUNDING", "load_backend", ] @@ -87,6 +91,10 @@ class DocumentRefused(ValueError): """ +#: The canonical form's refusal of two source keys that read as one key. +_KEY_COLLISION = "FLX-1007" + + class BlockRefused(ValueError): """The registry's parser refused the fidelity block a Simulation declares. @@ -99,6 +107,18 @@ def __init__(self, code: str, message: str) -> None: self.code = code +class BundleTextRefused(ValueError): + """The registry's exact-YAML reader or canonical form refused a bundle's text. + + The message names the file and, where the registry gives one, the line and + column, with the registry's ``FLX-nnnn`` code. It never quotes a value. + """ + + def __init__(self, code: str, message: str) -> None: + super().__init__(f"{message} ({code})") + self.code = code + + class MeasureRefused(ValueError): """The registry refused a record while forming a declared statistic. @@ -119,8 +139,10 @@ class DeclaredStatistic: ``shape`` holds the document keys that say which statistic it is beyond its definition and column: ``{"bins": [...]}`` (the declared edges, as decimal strings), ``{"levels": [...]}`` (the declared levels in cell order), - ``{"level": "0.5"}`` or nothing. ``columns`` is every column it reads, the - ``groupBy`` column included, and ``read_as`` the domain each is read in. + ``{"level": "0.5"}`` (and ``bins`` too, for a ``quantile/2``) or nothing. + ``columns`` is every column it reads, the ``groupBy`` column included, and + ``read_as`` the domain each is read in. ``edges`` are the declared bins + placed on the grid, for a histogram and a ``quantile/2``. """ identifier: str @@ -130,6 +152,7 @@ class DeclaredStatistic: shape: Mapping[str, Any] columns: Tuple[str, ...] read_as: Mapping[str, str] + edges: Tuple[int, ...] = () #: The registry's parsed declaration; only the registry's own seam reads it. _declaration: Any = field(default=None, repr=False, compare=False) @@ -167,13 +190,16 @@ class MeasuredGroup: statistic's quantities), with every count exact and no ``status``. The other fields hold what the disclosure rules read: ``cells`` (a histogram's bin or level counts), ``member_holders`` (the records each member of a support is - held by), ``quantile`` (witness, records strictly below it, records at or - below it) and, for a mean, the group's exact values and their grid points. + held by), ``quantile`` (a ``quantile/1``'s witness, records strictly below + it, records at or below it) and, for a mean, the group's exact values and + their grid points. ``statistic`` is the registry's measured group, which the + check for counts at edges (:meth:`StatisticsBackend.check_edge_counts`) reads. """ key: str document: Dict[str, Any] k: int + statistic: Any = field(default=None, repr=False, compare=False) cells: Tuple[int, ...] = () member_holders: Tuple[int, ...] = () quantile: Optional[Tuple[Optional[int], int, int]] = None @@ -198,6 +224,56 @@ class MeasuredStatistic: left_out_unread: bool = False +#: Why a count is withheld for its rounding, in the registry's own word +#: (``flux_stats.edge_counts.ROUNDING``; a test keeps the two equal). A caller +#: gives it as the status of a group or cell the profiler's own rounding withheld. +WITHHELD_ROUNDING = "rounding" + + +@dataclass(frozen=True) +class EdgeStatistic: + """One statistic of a column that counts records between edges, as measured. + + ``identifier`` is the definition's (``marginalHistogram/1``, ``levelHistogram/1``, + ``quantile/1`` or ``quantile/2``). ``edges`` are the encoded bin edges of a + histogram or a ``quantile/2``, ``levels`` a level histogram's declared levels, + ``groups`` the registry's measured groups (:attr:`MeasuredGroup.statistic`) and + ``left_out`` the rows a ``groupBy`` null leaves out of every group, measured as one + more group that is checked and never written. ``group_status`` and ``cell_status`` + hold what the profiler's own checks already withheld, by group key and per cell: + ``F`` or :data:`WITHHELD_ROUNDING`. + """ + + identifier: str + edges: Tuple[int, ...] + levels: Tuple[str, ...] + groups: Tuple[Any, ...] + group_by: Optional[str] = None + level: Optional[Fraction] = None + left_out: Any = None + group_status: Mapping[str, str] = field(default_factory=dict) + cell_status: Mapping[str, Tuple[str, ...]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class EdgeDecision: + """What the check for counts at edges decided about one :class:`EdgeStatistic`. + + ``group_status`` is ``F`` or ``C`` for each group key (the row left out is not + among them) and ``cell_status`` the same per cell of each published histogram + group. ``group_reason`` and ``cell_reason`` say why each withheld value is + withheld, in the registry's words, and are for the profiler's report alone. + ``rounded`` holds each published quantile group spelled as the profile writes + it, its counts already rounded. + """ + + group_status: Mapping[str, str] + cell_status: Mapping[str, Tuple[str, ...]] + group_reason: Mapping[str, str] + cell_reason: Mapping[str, Tuple[str, ...]] + rounded: Mapping[str, Dict[str, Any]] + + @runtime_checkable class MetricDefinition(Protocol): """One registered column metric (``count/1`` ... ``quantiles/1``). @@ -300,8 +376,40 @@ def measure( record the block's declaration refuses. """ - def quantile_ranks_ok(self, k: int, level: Fraction, below: int, at_or_below: int) -> bool: - """Whether a witness with these ranks, over *k* records, is the *level* quantile.""" + def check_edge_counts( + self, + statistics: Sequence[EdgeStatistic], + *, + total: int, + minimum_cell: int, + zeros_disclosive: bool, + round_count: Callable[[int], int], + ) -> Tuple[EdgeDecision, ...]: + """Apply the output check for counts at a column's edges to all its statistics at once. + + *statistics* are every histogram and quantile of one column, in declaration + order, and *total* the column's non-null record count. The result holds one + decision for each. Raises :class:`ValueError` for counts that cannot describe + one set of records. + """ + + def read_budget(self) -> Any: + """A fresh budget for reading one bundle's documents, to share across its files.""" + + def read_documents(self, text: str, *, name: str, budget: Any) -> Tuple[Mapping[str, Any], ...]: + """Every mapping document in *text*, read as the FLUX engine reads a bundle. + + A float is the decimal of its source text and a timestamp stays a string. The + read is bounded by *budget* (aliases, merge keys, digits, depth). Raises + :class:`BundleTextRefused` for text it refuses, naming *name*. + """ + + def canonicalise(self, value: Any, *, name: str) -> Any: + """*value* in the engine's canonical form (KERNEL-SPEC 2.1). + + Every key is an NFC string, every string NFC, a decimal its string spelling. + Raises :class:`BundleTextRefused`, naming *name*, for a value with no canonical form. + """ #: The most digits a number may need written out without an exponent: the @@ -387,7 +495,7 @@ class _FluxStatsBackend: def __init__(self) -> None: # The package first: when it is missing, this is the import that fails, # even with one of its submodules still in sys.modules. - from flux_stats import Parameters, definitions, metrics + from flux_stats import Parameters, canonical, definitions, edge_counts, exact_yaml, metrics from flux_stats.errors import FluxError from flux_stats.fixedpoint import Grid from flux_stats.profile import load_profile_text @@ -401,6 +509,9 @@ def __init__(self) -> None: self._category = category self._read_exact = read_exact self._value_domain = ValueDomain + self._edge_counts = edge_counts + self._exact_yaml = exact_yaml + self._canonical = canonical self._columns: Dict[Tuple[str, str], _Column] = {} self._grammar: Any = None @@ -439,6 +550,31 @@ def read_value(self, domain: str, value: Any, *, column: str, row_index: int) -> # A timestamp reads as a whole number of milliseconds since the epoch. return int(exact) if domain == "timestamp" else exact + def read_budget(self) -> Any: + return self._exact_yaml.ReadBudget() + + def read_documents(self, text: str, *, name: str, budget: Any) -> Tuple[Mapping[str, Any], ...]: + try: + documents: Tuple[Mapping[str, Any], ...] = self._exact_yaml.load_documents( + text, name=name, budget=budget + ) + except self._refused as error: + raise BundleTextRefused(error.code.code, error.detail or error.code.title) from None + return documents + + def canonicalise(self, value: Any, *, name: str) -> Any: + try: + return self._canonical.canonicalise(value) + except self._refused as error: + # FLX-1007 names the key two source keys read as, and a bundle key may + # hold anything, so only the code's title is said for it. + detail = ( + error.code.title + if error.code.code == _KEY_COLLISION + else (error.detail or error.code.title) + ) + raise BundleTextRefused(error.code.code, f"{name}: {detail}") from None + def check_document(self, text: str) -> None: try: self._load_profile_text(text) @@ -456,8 +592,8 @@ def _fidelity(self) -> Any: # By name, so a flux-stats that lacks the module raises here. fidelity = import_module("flux_stats.fidelity") reading = import_module("flux_stats.reading") - from flux_stats.fixedpoint import quantile_rank_ok - from flux_stats.registry import definition_for_kind + fixedpoint = import_module("flux_stats.fixedpoint") + from flux_stats.registry import definition as definition_named from flux_stats.spelling import render_exact except ImportError as error: missing = error.name or "flux_stats.fidelity" @@ -468,8 +604,8 @@ def _fidelity(self) -> Any: self._grammar = SimpleNamespace( fidelity=fidelity, reading=reading, - quantile_rank_ok=quantile_rank_ok, - definition_for_kind=definition_for_kind, + fixedpoint=fixedpoint, + definition_named=definition_named, render_exact=render_exact, ) return self._grammar @@ -491,8 +627,10 @@ def read_block( statistics: List[DeclaredStatistic] = [] left_out: List[Tuple[str, str]] = [] seen = set() - for declaration in parsed.properties: + for index, declaration in enumerate(parsed.properties): kind = declaration.kind.value + if declaration.bins: + self._refuse_off_grid_bins(grammar, parsed, index, declaration.bins) if kind == "correlation": first, second = declaration.columns left_out.append((f"correlation of {first!r} and {second!r}", _CORRELATION_LEFT_OUT)) @@ -514,6 +652,21 @@ def read_block( _block=parsed, ) + def _refuse_off_grid_bins(self, grammar: Any, parsed: Any, index: int, bins: Any) -> None: + """Refuse bins the fidelity check could not compare, before a record is read. + + The check bins a record exactly only where every edge is a grid point and the + edges strictly increase, and refuses any other (``FLX-5004``), so the profile + applies the registry's own rule. Edges that fall to one grid point would also + leave the output check over a column with no partition to read. The message + names the property and the edge's place, never a value. + """ + try: + grammar.fixedpoint.epsilon_marginal_histogram(parsed.grid, bins) + except self._refused as error: + detail = error.detail or error.code.title + raise BlockRefused(error.code.code, f"properties[{index}]: {detail}") from None + @staticmethod def _declared(grammar: Any, parsed: Any, declaration: Any) -> DeclaredStatistic: """One property of the parsed block as the statistic a profile carries for it.""" @@ -521,6 +674,7 @@ def _declared(grammar: Any, parsed: Any, declaration: Any) -> DeclaredStatistic: column: str = declaration.columns[0] group_by: Optional[str] = declaration.group_by shape: Dict[str, Any] = {} + edges: Tuple[int, ...] = () categorical = kind in ("support", "levelHistogram") if kind == "marginalHistogram": shape["bins"] = [grammar.render_exact(edge) for edge in declaration.bins] @@ -528,18 +682,26 @@ def _declared(grammar: Any, parsed: Any, declaration: Any) -> DeclaredStatistic: shape["levels"] = list(grammar.reading.level_cells(parsed.columns[column])) elif kind == "quantile": shape["level"] = grammar.render_exact(declaration.level) + if declaration.bins: # a quantile that declares bins is quantile/2 + shape["bins"] = [grammar.render_exact(edge) for edge in declaration.bins] + if declaration.bins and kind in ("marginalHistogram", "quantile"): + # As the fidelity check places them: it forms a group on these edges. + edges = tuple( + parsed.grid.quantise(edge, subject="bin edge") for edge in declaration.bins + ) names = (column,) if group_by is None else (column, group_by) read_as = {column: "category" if categorical else parsed.columns[column].domain.value} if group_by is not None: read_as[group_by] = "category" return DeclaredStatistic( - identifier=grammar.definition_for_kind(declaration.kind).identifier, + identifier=grammar.fidelity.definition_for_property(declaration).identifier, kind=kind, column=column, group_by=group_by, shape=shape, columns=names, read_as=read_as, + edges=edges, _declaration=declaration, ) @@ -599,13 +761,18 @@ def _measured( rows, indices, key, declaration, parsed.columns, parsed.grid ) kind = statistic.kind - spelled: Dict[str, Any] = state.to_document(kind) - group = MeasuredGroup(key=key, document=spelled, k=state.k) + # By definition, not by kind: a kind spells a quantile as quantile/1, and + # refuses a group a quantile/2 formed. + definition = grammar.fidelity.definition_for_property(declaration) + spelled: Dict[str, Any] = state.to_document(definition) + group = MeasuredGroup(key=key, document=spelled, k=state.k, statistic=state) if kind in ("marginalHistogram", "levelHistogram"): return replace(group, cells=tuple(state.cell_counts(kind))) if kind == "support": return replace(group, member_holders=tuple(state.support_counts)) if kind == "quantile": + if definition.binned: + return group ranks = (state.quantile_witness, state.quantile_below, state.quantile_at_or_below) return replace(group, quantile=ranks) # A mean: the same indices, read as the block reads them, so the null policy @@ -620,15 +787,85 @@ def _measured( ) return replace(group, values=tuple(values), encoded=tuple(encoded)) - def quantile_ranks_ok(self, k: int, level: Fraction, below: int, at_or_below: int) -> bool: + def check_edge_counts( + self, + statistics: Sequence[EdgeStatistic], + *, + total: int, + minimum_cell: int, + zeros_disclosive: bool, + round_count: Callable[[int], int], + ) -> Tuple[EdgeDecision, ...]: + edges = self._edge_counts grammar = self._fidelity() + asked = [] + for each in statistics: + groups = tuple(each.groups) + group_status = dict(each.group_status) + if each.left_out is not None: + # The row a groupBy null leaves out: counted, never published, and under a + # key no declared level can be. + groups += (replace(each.left_out, group=_LEFT_OUT_KEY),) + group_status[_LEFT_OUT_KEY] = _LEFT_OUT_STATUS + asked.append( + edges.EdgeStatistic( + identifier=each.identifier, + edges=each.edges, + levels=each.levels, + groups=groups, + group_by=each.group_by, + level=each.level, + group_status=group_status, + cell_status=dict(each.cell_status), + ) + ) try: - proved: bool = grammar.quantile_rank_ok( - count=k, level=level, count_strictly_below=below, count_at_or_below=at_or_below + decisions = edges.check_edge_counts( + asked, + total=total, + minimum_cell=minimum_cell, + zeros_disclosive=zeros_disclosive, + round_count=round_count, ) - except self._refused: - return False - return proved + except self._refused as error: + # The registry's words may give counts; its code alone says what it refused. + raise ValueError(error.code.code) from None + return tuple( + self._decided(grammar, each, decision) + for each, decision in zip(statistics, decisions, strict=True) + ) + + @staticmethod + def _decided(grammar: Any, each: EdgeStatistic, decision: Any) -> EdgeDecision: + """The registry's decision without the row left out, with each quantile group spelled.""" + definition = grammar.definition_named(each.identifier) + + def kept(values: Mapping[str, Any]) -> Dict[str, Any]: + return {key: value for key, value in values.items() if key != _LEFT_OUT_KEY} + + return EdgeDecision( + group_status=kept(decision.group_status), + cell_status=kept(decision.cell_status), + group_reason=kept(decision.group_reason), + cell_reason=kept(decision.cell_reason), + rounded={ + key: _spelled(group, definition) + for key, group in decision.rounded.items() + if key != _LEFT_OUT_KEY + }, + ) + + +#: The key and status the row a ``groupBy`` null leaves out is checked under: no +#: declared level can start with a NUL, and the status is never ``F``. +_LEFT_OUT_KEY = "\x00left out" +_LEFT_OUT_STATUS = "left out" + + +def _spelled(group: Any, definition: Any) -> Dict[str, Any]: + """A rounded quantile group as the profile writes it.""" + document: Dict[str, Any] = group.to_document(definition) + return document #: Why a correlation is not answered (founder decision, 2026-11-18). @@ -652,7 +889,9 @@ def load_backend() -> StatisticsBackend: #: The flux-stats range ``[profile]`` names in ``pyproject.toml``. The CLI cannot read #: that file once installed, so the install message repeats it; #: ``tests/profiler/test_flux_stats_pin.py`` keeps the two equal. -FLUX_STATS_REQUIREMENT = "flux-stats>=0.6.2,<0.7" +#: 0.7.0 is the first flux-stats release that holds ``edge_counts``, ``exact_yaml`` and +#: ``quantile/2``. +FLUX_STATS_REQUIREMENT = "flux-stats[yaml]>=0.7.0,<0.8" #: The extra carries the range, and pip replaces a copy outside it (the 0.0.1 #: placeholder, or a release newer than this module knows). FLUX_STATS_INSTALL_COMMAND = "pip install 'data-product-forge[profile]'" diff --git a/fluid_build/profiler/bundle.py b/fluid_build/profiler/bundle.py index a72b12c3..7ed71c41 100644 --- a/fluid_build/profiler/bundle.py +++ b/fluid_build/profiler/bundle.py @@ -18,33 +18,28 @@ (:meth:`fluid_build.profiler._stats.StatisticsBackend.read_block`), so the profiler implements no grammar of its own and imports no registry here. -The file is read the way the FLUX engine reads it, and the block is returned in -the engine's canonical form, so the registry's parser sees what the engine's -sees. This is a port, not an import (the profiler never imports the engine): - -* the loader is ``flux_engine/seam/resolve.py`` ``ExactLoader``: a float scalar - is its source text as a ``Decimal``, a timestamp stays a string, and an - integer too long to form is refused; -* the canonical form is ``flux_engine/core/canonical.py`` ``coerce_key``, - ``canonicalise`` and ``_render_decimal`` (KERNEL-SPEC 2.1): every mapping key - is an NFC string (``True`` is ``"true"``, ``None`` is ``"null"``, an integer - its digits), every string is NFC, a ``Decimal`` becomes its string spelling, - and a float key or an integer outside the JCS safe range is refused. The - engine parses the block from that form (``stages/pipeline.py``). +The text is read the way the FLUX engine reads it, by flux-stats' exact-YAML +reader (``flux_stats.exact_yaml``) and canonical form (``flux_stats.canonical``), +both reached through the backend seam: a float scalar is its source text as a +decimal, a timestamp stays a string, an integer too long to form is refused, and +the block is returned in the engine's canonical form (KERNEL-SPEC 2.1), so the +registry's parser sees what the engine's sees. The reader bounds what a document +may write out (aliases, merge keys, digits, depth), with one budget for every +document of the bundle. A refusal names the file and the keys it looked for, never a value from a record. Nothing in a bundle is a record, but an operator's terminal output -travels, so a YAML error is reduced to its line number. +travels, so a YAML error is reduced to what the registry's reader reports. """ from __future__ import annotations -import sys import unicodedata from dataclasses import dataclass -from decimal import Decimal, InvalidOperation from pathlib import Path -from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Dict, List, Mapping, Optional, Tuple + +from fluid_build.profiler._stats import BundleTextRefused, StatisticsBackend __all__ = [ "BUNDLE_NAMESPACE", @@ -64,33 +59,6 @@ _MAX_BYTES = 4 * 1024 * 1024 -#: The largest integer JCS (RFC 8785) writes exactly: 2**53 - 1. -_JCS_MAX_SAFE_INTEGER = 2**53 - 1 - -#: The most digits an integer may have: CPython's own limit on int/text conversion. -_MOST_INTEGER_DIGITS: int = sys.int_info.default_max_str_digits - -_INTEGER_TAG = "tag:yaml.org,2002:int" -_FLOAT_TAG = "tag:yaml.org,2002:float" -_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" -_MERGE_TAG = "tag:yaml.org,2002:merge" - -#: The most digits a number in a block may need written out: the engine's rule for a -#: declared number, flux-stats' ``MOST_DIGITS``, counted as its -#: ``reading.digits_written_out`` counts (:func:`_digits_written_out`). It is checked -#: before a decimal is written out, since writing ``1E+300000000`` out takes gigabytes. -_MOST_NUMBER_DIGITS = 1000 - -#: The most values, and the deepest nesting, a block may hold once written out. YAML -#: reads an alias as a shared reference, so a kilobyte of aliases can stand for -#: billions of values; a real block holds a few hundred, a few levels deep. -_MOST_BLOCK_VALUES = 100_000 -_DEEPEST_BLOCK = 64 -#: The most characters of a key a refusal's path names: the path is built for every -#: value, so an aliased long key must not be copied whole at each level. -_KEY_IN_PATH = 40 - - class BundleError(Exception): """A bundle that cannot give a fidelity block. ``event`` is a stable code.""" @@ -126,228 +94,25 @@ def _documents(directory: Path) -> List[Path]: ] -class _Merges: - """The pairs YAML merge keys may still copy while one bundle is read.""" - - def __init__(self) -> None: - self.left = _MOST_BLOCK_VALUES - - -def _exact_loader(name: str, merges: _Merges) -> Any: - """The engine's ``ExactLoader``, built on first use so PyYAML stays off the cold path. - - Its merge keys draw on *merges*, which every document of one bundle shares. - """ - import yaml - - def merged_parts(value_node: Any) -> List[Any]: - """The mappings a merge key's value names: one mapping, or a list of them.""" - if isinstance(value_node, yaml.MappingNode): - return [value_node] - if isinstance(value_node, yaml.SequenceNode): - return [part for part in value_node.value if isinstance(part, yaml.MappingNode)] - return [] - - class ExactLoader(yaml.SafeLoader): - """A float is its source text as a Decimal, a timestamp a string, a long int refused.""" - - def flatten_mapping(self, node: Any) -> None: - # A merge key (``<<``) copies its mappings' pairs while the document is - # built, before _canonicalise counts a value, and a chain of merges doubles - # at each level. So every pair a merge copies, in any document of the - # bundle, is counted against one bound before the copy is made. - for key_node, value_node in node.value: - if key_node.tag != _MERGE_TAG: - continue - for part in merged_parts(value_node): - self.flatten_mapping(part) - merges.left -= len(part.value) - if merges.left < 0: - raise refuse( - node, - f"merge keys (<<) copy more than {_MOST_BLOCK_VALUES} pairs " - "in the bundle", - ) - super().flatten_mapping(node) - - def refuse(node: Any, why: str) -> BundleError: - mark = node.start_mark - return BundleError( - "profile_bundle_unreadable", - f"{name} line {mark.line + 1}, column {mark.column + 1}: {why}", - {"next": "write the value as the FLUX engine reads it, or name another bundle"}, - ) - - def construct_float(loader: Any, node: Any) -> Decimal: - text = str(loader.construct_scalar(node)).replace("_", "") - if ":" in text: - raise refuse(node, "a sexagesimal number has no decimal spelling; write plain decimal") - try: - value = Decimal(text) - except InvalidOperation: - raise refuse(node, "a scalar read as a float is not a decimal number") from None - if not value.is_finite(): - raise refuse(node, "an infinite or not-a-number value is not allowed") - return value - - def construct_int(loader: Any, node: Any) -> int: - too_long = f"an integer of more than {_MOST_INTEGER_DIGITS} digits is too long to read" - digits = sum(ch.isdigit() for ch in str(loader.construct_scalar(node))) - if digits > _MOST_INTEGER_DIGITS: - raise refuse(node, too_long) - try: - value = int(loader.construct_yaml_int(node)) - except (ValueError, IndexError): - raise refuse(node, "a scalar read as an integer has no integer value") from None - if abs(value) >= 10**_MOST_INTEGER_DIGITS: - raise refuse(node, too_long) - return value - - ExactLoader.add_constructor(_FLOAT_TAG, construct_float) - ExactLoader.add_constructor(_INTEGER_TAG, construct_int) - - def construct_timestamp(loader: Any, node: Any) -> Any: - return loader.construct_scalar(node) - - ExactLoader.add_constructor(_TIMESTAMP_TAG, construct_timestamp) - return ExactLoader - - def _nfc(text: str) -> str: return unicodedata.normalize("NFC", text) -def _digits_written_out(value: Decimal) -> int: - """The digits *value* needs written out, as flux-stats' ``digits_written_out`` counts.""" - digits = len(value.as_tuple().digits) - exponent = value.adjusted() - digits + 1 - if not value: - return 1 # zero, however its exponent is spelled - if exponent >= 0: - return digits + exponent - return max(digits, -exponent) - - -def _render_decimal(value: Decimal, name: str, where: str) -> str: - """The declared scale is kept; negative zero is folded onto the positive spelling.""" - if _digits_written_out(value) > _MOST_NUMBER_DIGITS: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} holds a number of more than {_MOST_NUMBER_DIGITS} digits " - "written out, which FLUX does not read", - {"next": "write a shorter number"}, - ) - rendered = format(value, "f") - if rendered.startswith("-") and value == 0: - rendered = rendered[1:] - return rendered - - -def _coerce_key(key: Any, name: str, where: str) -> str: - if isinstance(key, str): - return _nfc(key) - if isinstance(key, bool): - return "true" if key else "false" - if isinstance(key, float): - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} has a float key, which has no canonical spelling", - {"next": "quote the key so it is a string"}, - ) - if isinstance(key, int): - return str(key) - if key is None: - return "null" - if isinstance(key, Decimal): - return _render_decimal(key, name, where) - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} has a key of class {type(key).__name__!r}", - {"next": "quote the key so it is a string"}, - ) - - -class _Budget: - """The values left to write one block out with.""" - - def __init__(self) -> None: - self.left = _MOST_BLOCK_VALUES - - def spend(self, name: str, where: str, depth: int) -> None: - """Count one value at *depth*; refuse a block past either bound.""" - self.left -= 1 - if self.left < 0: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} holds more than {_MOST_BLOCK_VALUES} values once its " - "YAML aliases are written out", - {"next": "write the block without aliases, or name another bundle"}, - ) - if depth > _DEEPEST_BLOCK: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} nests deeper than {_DEEPEST_BLOCK} levels", - {"next": "write the block without aliases, or name another bundle"}, - ) - - -def _canonicalise( - value: Any, name: str, where: str, budget: Optional[_Budget] = None, depth: int = 0 -) -> Any: - """*value* as JSON-ready Python (KERNEL-SPEC 2.1), mirroring the engine's ``canonicalise``. - - Each value written out costs one of *budget*, so aliases cannot make the walk - write out more than :data:`_MOST_BLOCK_VALUES` values or go deeper than - :data:`_DEEPEST_BLOCK` levels. - """ - budget = budget if budget is not None else _Budget() - budget.spend(name, where, depth) - if isinstance(value, bool): - return value - if isinstance(value, int): - if abs(value) > _JCS_MAX_SAFE_INTEGER: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} holds an integer outside the range 2^53 - 1 that FLUX reads", - {"next": "write a smaller integer, or quote it as a string"}, - ) - return value - if isinstance(value, Decimal): - return _render_decimal(value, name, where) - if isinstance(value, str): - return _nfc(value) - if value is None: - return None - if isinstance(value, Mapping): - canonical: Dict[str, Any] = {} - for raw_key, raw_value in value.items(): - key = _coerce_key(raw_key, name, where) - if key in canonical: - raise BundleError( - "profile_bundle_unreadable", - f"{name}: {where} has two keys that both read as {key!r}", - {"next": "keep one of them"}, - ) - step = key if len(key) <= _KEY_IN_PATH else f"{key[:_KEY_IN_PATH]}..." - canonical[key] = _canonicalise(raw_value, name, f"{where}.{step}", budget, depth + 1) - return {key: canonical[key] for key in sorted(canonical)} - if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray)): - return [_canonicalise(item, name, where, budget, depth + 1) for item in value] - raise BundleError( +def _unreadable(message: str) -> BundleError: + return BundleError( "profile_bundle_unreadable", - f"{name}: {where} holds a value of class {type(value).__name__!r}", - {"next": "write plain YAML scalars, lists and mappings"}, + message, + {"next": "fix the document, or name another bundle directory"}, ) -def _load(path: Path, merges: _Merges) -> Tuple[Mapping[str, Any], ...]: - """Every mapping document in *path*, as the engine's ``load_documents_exact`` reads it. +def _load(path: Path, backend: StatisticsBackend, budget: Any) -> Tuple[Mapping[str, Any], ...]: + """Every mapping document in *path*, as flux-stats' ``load_documents`` reads it. A file may hold more than one ``---`` document; a :class:`BundleError` names the - file and the line of a document that cannot be read. + file, and the line where the registry's reader gives one, of a document that cannot + be read. """ - import yaml - try: if path.stat().st_size > _MAX_BYTES: raise BundleError( @@ -362,27 +127,12 @@ def _load(path: Path, merges: _Merges) -> Tuple[Mapping[str, Any], ...]: "profile_bundle_unreadable", f"{path.name} cannot be read: {reason}" ) from None try: - documents = list( - yaml.load_all(text, Loader=_exact_loader(path.name, merges)) # nosec B506 - SafeLoader - ) - except RecursionError: - raise BundleError( - "profile_bundle_unreadable", - f"{path.name} nests too deeply to read", - {"next": "fix the document, or name another bundle directory"}, - ) from None - except yaml.YAMLError as exc: - mark = getattr(exc, "problem_mark", None) - where = f" at line {mark.line + 1}" if mark is not None else "" - raise BundleError( - "profile_bundle_unreadable", - f"{path.name} is not valid YAML{where}", - {"next": "fix the document, or name another bundle directory"}, - ) from None - return tuple(document for document in documents if isinstance(document, Mapping)) + return backend.read_documents(text, name=path.name, budget=budget) + except BundleTextRefused as exc: + raise _unreadable(str(exc)) from None -def read_bundle_block(directory: Path, expose_id: str) -> BundleBlock: +def read_bundle_block(directory: Path, expose_id: str, backend: StatisticsBackend) -> BundleBlock: """The fidelity block the bundle's one Simulation declares for *expose_id*. Refuses, each with its own event, a directory with no Simulation or more than @@ -390,9 +140,9 @@ def read_bundle_block(directory: Path, expose_id: str) -> BundleBlock: not a mapping. """ simulations: List[Tuple[Path, Mapping[str, Any]]] = [] - merges = _Merges() # one bound for every document of the bundle + budget = backend.read_budget() # one bound for every document of the bundle for path in _documents(directory): - for document in _load(path, merges): + for document in _load(path, backend, budget): if document.get("kind") == "Simulation": simulations.append((path, document)) if not simulations: @@ -436,5 +186,8 @@ def read_bundle_block(directory: Path, expose_id: str) -> BundleBlock: f"{path.name}: {where} is not a mapping", {"next": "declare the fidelity block as FLUX's fidelity grammar spells it"}, ) - block = _canonicalise(block, path.name, where) + try: + block = backend.canonicalise(block, name=f"{path.name}: {where}") + except BundleTextRefused as exc: + raise _unreadable(str(exc)) from None return BundleBlock(raw=block, simulation=path.name, expose_id=expose_id) diff --git a/fluid_build/profiler/disclosure.py b/fluid_build/profiler/disclosure.py index 04c5525f..44106fe7 100644 --- a/fluid_build/profiler/disclosure.py +++ b/fluid_build/profiler/disclosure.py @@ -29,23 +29,18 @@ also gives the total of what the values lose below their grid points, so that total is checked as a sum as well (:func:`check_mean`). -A minimum, a maximum or a quantile is one record's value in most data. The -DwB/ESSnet "Guidelines for Output Checking" (class 3, maxima, minima and -percentiles) do not release one by rule of thumb, and clear one when more than the -threshold of units hold it. The profiler takes that reading: such a value is -published only when at least ``minimumCell`` records hold exactly that value -(founder decision, 2026-11-16). A quantile's two ranks are counts too, so the -records below its witness and the records above it must each be at least -``minimumCell`` (FLUX ADR 0012's three cells, founder decision 2026-11-18): a rank -of 3 would say that three records lie below the witness. None is allowed only where -zeros are not disclosive, since it says no record lies between the witness and the -end of the range. A published minimum or maximum says that no record lies beyond it, -and no check withholds it for that (FLUX ADR 0012, amended 2026-11-18). This leaves -the checks inconsistent at the ends of the range: where zeros are disclosive, a -quantile whose witness is the column's smallest value is withheld because no record -lies below the witness, though a published minimum already shows that, and one whose -witness is the largest value is withheld because no record lies above it, though a -published maximum already shows that. +A minimum, a maximum or a column's ``quantiles`` metric is one record's value in +most data. The DwB/ESSnet "Guidelines for Output Checking" (class 3, maxima, +minima and percentiles) do not release one by rule of thumb, and clear one when +more than the threshold of units hold it. The profiler takes that reading: such a +value is published only when at least ``minimumCell`` records hold exactly that +value (founder decision, 2026-11-16). A published minimum or maximum says that no +record lies beyond it, and no check withholds it for that (FLUX ADR 0012, amended +2026-11-18). This leaves the checks inconsistent at the ends of the range: where +zeros are disclosive, a quantile whose witness is the column's smallest value is +withheld because no record lies below the witness, though a published minimum +already shows that, and one whose witness is the largest value is withheld because +no record lies above it, though a published maximum already shows that. A verdict names the check that fired with an SDMX ``CL_CONF_STATUS`` 1.4 code: ``A`` for a small count (a zero cell, a cell below the threshold, or a value too @@ -55,20 +50,26 @@ another cannot be recovered (secondary suppression, :func:`suppress_secondary`). A withheld value is written with that code and no value, except in a count table or a quantile (:data:`ONE_CODE`, PROFILE-SPEC §3), whose withheld groups and cells -are all written ``C``, whatever withheld them (:func:`written_status`, FLUX ADR -0012): a code that named the check would tell a reader a range, ``A`` that a cell -holds fewer than ``minimumCell`` records, and a total the reader knows can turn -the range into the count. The profiler's summary still counts them by the check, as -ACRO's outcome table keeps each suppressed cell's reasons apart from the table it -releases. - -The groups of a statistic the Simulation's fidelity block declares are checked -group by group, each as a cell of a table of records (the ESSnet/DwB output +are all written ``C``, whatever withheld them: a code that named the check would +tell a reader a range, ``A`` that a cell holds fewer than ``minimumCell`` records, +and a total the reader knows can turn the range into the count. The profiler's +summary still counts them by the check, as ACRO's outcome table keeps each +suppressed cell's reasons apart from the table it releases. + +The groups of a mean or a support the Simulation's fidelity block declares are +checked group by group, each as a cell of a table of records (the ESSnet/DwB output checking guidelines' frequency-table and magnitude-table classes, which ACRO's ``safe_threshold`` and ``safe_nk`` checks implement): :func:`check_group_count` -for the group's record count, :func:`publish_cells` for a histogram's cells, -:func:`check_support` for a support's members and :func:`check_quantile` for a -quantile's witness. +for the group's record count, :func:`check_support` for a support's members. + +The counts a histogram, a level histogram or a quantile publishes at the edges of +its column are not checked here. FLUX ADR 0012 sets one rule for all of them, which +flux-stats applies (``flux_stats.edge_counts.check_edge_counts``, called through +:mod:`fluid_build.profiler._stats`): one declared partition, three cells for a +quantile, a group's size, no small cell a reader can form by adding or subtracting +published counts, and only then the rounding. What it leaves readable is its own to +say. The profiler passes it every such statistic of a column in one call, with this +module's :func:`round_count`. What the group checks leave readable, on purpose: @@ -80,13 +81,6 @@ it is withheld too, under ``D``, so the withheld one cannot be recovered by differencing against the ungrouped figures. The records a ``groupBy`` null leaves out of every group count as one more group there, never written. -* A group withheld for its size withholds every group of its groupBy's - histograms and quantiles too (the runner's sibling rule, FLUX ADR 0012), and a - quantile is published only as the one histogram or quantile of its column. - A withheld cell of a histogram is not secondarily suppressed: the rounded - counts still bound it, somewhere inside 0 to 9 records and never above the - threshold, until forge-cli applies FLUX's ``check_edge_counts``, which a - flux-stats release has to carry first. Its ``C`` adds nothing to that bound. Where this differs from ACRO 1.1.0, on purpose: @@ -133,7 +127,6 @@ "SECONDARY", "SMALL_COUNT", "WITHHELD", - "Cells", "DisclosureRule", "Exact", "Verdict", @@ -142,15 +135,12 @@ "check_magnitude", "check_mean", "check_order_statistic", - "check_quantile", "check_rounding", "check_sum", "check_support", - "publish_cells", "round_count", "sample_size", "suppress_secondary", - "written_status", ] # SDMX ``CL_CONF_STATUS`` 1.4 codes, with PROFILE-SPEC §3's paraphrase of each @@ -174,27 +164,12 @@ SECONDARY = "D" #: The declared statistics whose every withheld group and cell is written ``C``, whatever -#: withheld it (:func:`written_status`; PROFILE-SPEC §3): the count tables, a histogram -#: over bins and one over levels, and a quantile. +#: withheld it (PROFILE-SPEC §3; the runner writes them from the decisions of flux-stats' +#: ``check_edge_counts``): the count tables, a histogram over bins and one over levels, +#: and a quantile. ONE_CODE = frozenset({"marginalHistogram", "levelHistogram", "quantile"}) -def written_status(status: str, kind: str) -> str: - """The status a document writes for a value that a check gave *status*. - - *kind* is the declared property the value belongs to (``mean``, ``quantile``, - ``marginalHistogram`` and so on). A value of a count table or a quantile that - any check withholds is written ``C``, so its status says nothing about its - size (FLUX ADR 0012, PROFILE-SPEC §3): ``A`` would say that a cell holds fewer - than ``minimumCell`` records, and cells of 10, 9, 9 and 10 records written - ``F``, ``A``, ``A``, ``F`` give both withheld cells as 9 to a reader who knows - the total. Every other value keeps the code of the check that withheld it. - """ - if status != FREE and kind in ONE_CODE: - return WITHHELD - return status - - #: The largest count a profile can carry: 2^53 - 1, as RFC 8785 serialises it. MAX_COUNT = 2**53 - 1 @@ -519,46 +494,6 @@ def check_group_count(k: int, rule: DisclosureRule) -> Verdict: return verdict -@dataclass(frozen=True) -class Cells: - """A histogram group's cells after the checks. - - ``verdict`` is :data:`PUBLISHED`, or ``A`` when every cell is withheld and the - group goes whole. ``statuses`` holds the code of the check that decided each - cell, which a document writes through :func:`written_status`, and ``counts`` - the published count of each, rounded, or ``None`` where the cell is withheld - (PROFILE-SPEC §3: a cell is null exactly when its status is not ``F``). - """ - - verdict: Verdict - statuses: Tuple[str, ...] - counts: Tuple[Optional[int], ...] - - @property - def any_withheld(self) -> bool: - return any(status != FREE for status in self.statuses) - - -def publish_cells(counts: Sequence[int], rule: DisclosureRule) -> Cells: - """Check each cell of a histogram group as a count, and round the ones that pass. - - A cell is a frequency-table cell: a zero is withheld when the rule holds zeros - disclosive, one below ``minimumCell`` is withheld, and so is a positive count - that rounds to zero (:func:`check_count`, then :func:`check_rounding`), each - with ``A``. A published cell is rounded to a multiple of the rounding base. A - group whose every cell is withheld says nothing, so it is withheld whole. - """ - statuses: List[str] = [] - shown: List[Optional[int]] = [] - for count in counts: - verdict = check_group_count(count, rule) - statuses.append(verdict.status) - shown.append(round_count(count, rule) if verdict.published else None) - if statuses and all(status != FREE for status in statuses): - return Cells(Verdict(SMALL_COUNT, "every cell is withheld"), tuple(statuses), tuple(shown)) - return Cells(PUBLISHED, tuple(statuses), tuple(shown)) - - def check_support(holders: Sequence[int], rule: DisclosureRule) -> Verdict: """Check a support: every member must be held by at least ``minimumCell`` records. @@ -576,72 +511,6 @@ def check_support(holders: Sequence[int], rule: DisclosureRule) -> Verdict: return PUBLISHED -def check_quantile( - k: int, - below: int, - at_or_below: int, - rule: DisclosureRule, - proves: Callable[[int, int, int], bool], -) -> Tuple[Verdict, int, int]: - """Check a quantile group's witness and its ranks; return the verdict and the rounded ranks. - - *below* and *at_or_below* are the records strictly below the witness and at or - below it, so ``at_or_below - below`` records hold it and ``k - at_or_below`` - lie above it. In order, the first that fires decides, always with ``A``: - - 1. fewer than ``minimumCell`` records hold the witness - (:func:`check_order_statistic`, as a minimum or a maximum is checked); - 2. the records below the witness, or above it, are fewer than ``minimumCell``: - on unrounded counts, the three cells the ranks state each hold at least - ``minimumCell`` records (FLUX ADR 0012's three cells), since a rank of 3 says - that three records lie below the witness, rounded or not. None below or - above is allowed only where the rule does not hold zeros disclosive; - 3. a rank is positive and rounds to zero (:func:`check_rounding`); - 4. the rounded ``k`` and ranks no longer prove the witness: *proves* is the - registry's rank check on them, because a published group is read back with - the rounded numbers, and a rank rounded across the witness's index would - publish a witness the numbers contradict. - - The ranks are published rounded, as every count is. The returned ranks are - only meaningful when the verdict is published. - """ - for count in (k, below, at_or_below): - _require_count(count) - if not below <= at_or_below <= k: - raise ValueError("a quantile's ranks lie from 0 to its k, the lower one first") - verdict = check_order_statistic(at_or_below - below, rule) - if verdict.published: - verdict = _check_outer_cells(below, k - at_or_below, rule) - if verdict.published: - for rank in (below, at_or_below): - verdict = check_rounding(rank, rule) - if not verdict.published: - break - shown = (round_count(below, rule), round_count(at_or_below, rule)) - if verdict.published and not proves(round_count(k, rule), shown[0], shown[1]): - verdict = Verdict(SMALL_COUNT, "the rounded ranks no longer prove the witness") - return verdict, shown[0], shown[1] - - -def _check_outer_cells(below: int, above: int, rule: DisclosureRule) -> Verdict: - """The records below a quantile's witness and above it: each enough, or none where allowed. - - None below or above says that no record lies between the witness and the end of - the range, a cell of no records, so it is allowed only where the rule does not hold - zeros disclosive (FLUX ADR 0012). - """ - for count, side in ((below, "below"), (above, "above")): - if count == 0 and rule.zeros_disclosive: - return Verdict( - SMALL_COUNT, f"no record lies {side} the value, and the rule holds zeros disclosive" - ) - if 0 < count < rule.minimum_cell: - return Verdict( - SMALL_COUNT, f"fewer than {rule.minimum_cell} records lie {side} the value" - ) - return PUBLISHED - - def suppress_secondary(verdicts: Sequence[Verdict]) -> Tuple[Verdict, ...]: """Withhold every published group of a statistic when any of its groups is withheld. diff --git a/fluid_build/profiler/document.py b/fluid_build/profiler/document.py index eff96844..f4c309a3 100644 --- a/fluid_build/profiler/document.py +++ b/fluid_build/profiler/document.py @@ -1,7 +1,7 @@ # Copyright 2025 Agentics Transformation Limited # SPDX-License-Identifier: Apache-2.0 -"""The profile document: PROFILE-SPEC format 1-0-0, its canonical bytes and digest. +"""The profile document: PROFILE-SPEC formats 1-0-0 and 1-1-0, its canonical bytes and digest. flux-engine ``docs/spec/PROFILE-SPEC.md`` §1 fixes the shape: the OpenLineage facet envelope (``_producer``, ``_schemaURL``), the grid (``encoding``), the @@ -15,6 +15,10 @@ digest from one implementation. The file holds exactly those bytes, so its sha256 is the digest a Simulation pins. +A profile names format 1-0-0, unless it holds a ``quantile/2`` statistic, which only +format 1-1-0 carries (:func:`schema_url_for`): a profile without one keeps the bytes it +had. + A document that breaks a rule is refused here, before anything is written. The refusal names where in the document the problem is, never a value. """ @@ -33,16 +37,26 @@ "MAX_FRACTIONAL_BITS", "PRODUCER_REPOSITORY", "SCHEMA_URL", + "SCHEMA_URL_1_1_0", "build_document", "canonical_bytes", "check_fractional_bits", "digest_of", "producer_uri", + "schema_url_for", ] -#: The format this module writes. A later format is a new URL (PROFILE-SPEC §8). +#: The format a profile names unless it needs the next one. A later format is a new URL +#: (PROFILE-SPEC §8). SCHEMA_URL = "https://agenticstiger.github.io/flux/engine/spec/profile/1-0-0/FluxProfile.json" +#: Format 1-1-0 adds the ``quantile/2`` statistic and changes nothing 1-0-0 says +#: (``flux_stats.profile.SCHEMA_URL_1_1_0``; a test keeps the two equal). +SCHEMA_URL_1_1_0 = "https://agenticstiger.github.io/flux/engine/spec/profile/1-1-0/FluxProfile.json" + +#: The statistics only format 1-1-0 holds. +_ADDED_IN_1_1_0 = frozenset({"quantile/2"}) + #: ``_producer`` names this repository, at the release that wrote the profile. PRODUCER_REPOSITORY = "https://github.com/Agenticstiger/forge-cli" @@ -73,6 +87,16 @@ def producer_uri(version: str) -> str: return f"{PRODUCER_REPOSITORY}/tree/v{version}/fluid_build/profiler" +def schema_url_for(tables: Mapping[str, Mapping[str, Any]]) -> str: + """The format *tables* need: 1-1-0 when any statistic is ``quantile/2``, else 1-0-0.""" + for table in tables.values(): + for column in table.get("columns", {}).values(): + for statistic in column.get("statistics", ()): + if statistic.get("definition") in _ADDED_IN_1_1_0: + return SCHEMA_URL_1_1_0 + return SCHEMA_URL + + def build_document( *, producer: str, @@ -80,13 +104,13 @@ def build_document( tables: Mapping[str, Mapping[str, Any]], fractional_bits: int = DEFAULT_FRACTIONAL_BITS, ) -> Dict[str, Any]: - """Assemble a format 1-0-0 document from tables already checked by *rule*.""" + """Assemble a document from tables already checked by *rule*, in the format they need.""" check_fractional_bits(fractional_bits) if not tables: raise DocumentError("tables holds at least one table") return { "_producer": producer, - "_schemaURL": SCHEMA_URL, + "_schemaURL": schema_url_for(tables), "encoding": {"fractionalBits": fractional_bits, "rounding": "floor"}, "disclosureRules": {rule.name: rule.to_document()}, "tables": {expose_id: dict(table) for expose_id, table in tables.items()}, diff --git a/fluid_build/profiler/runner.py b/fluid_build/profiler/runner.py index f233c197..91d7aed6 100644 --- a/fluid_build/profiler/runner.py +++ b/fluid_build/profiler/runner.py @@ -18,8 +18,9 @@ With a bundle (:mod:`fluid_build.profiler.bundle`) it forms, besides, each statistic the Simulation's fidelity block declares (a histogram, a support, a quantile, a grouped mean), from the same one read of the table. -4. :mod:`fluid_build.profiler.disclosure` decides what each metric and statistic - may publish, group by group. +4. :mod:`fluid_build.profiler.disclosure` decides what each metric, mean and support + may publish, group by group, and the registry's ``check_edge_counts`` (FLUX ADR + 0012) what the histograms and quantiles of one column may publish together. 5. :mod:`fluid_build.profiler.document` writes the canonical document, the registry reads it back, and its digest is taken. @@ -42,9 +43,12 @@ from fluid_build.policy._common import iter_exposes from fluid_build.profiler import disclosure, document, sources from fluid_build.profiler._stats import ( + WITHHELD_ROUNDING, BlockRefused, DeclaredBlock, DeclaredStatistic, + EdgeDecision, + EdgeStatistic, MeasuredGroup, MeasuredStatistic, MeasureRefused, @@ -172,8 +176,8 @@ class ProfileResult: plan: ExposePlan published: int #: Withheld values per ``CL_CONF_STATUS`` code of the check that withheld them: a - #: histogram's withheld groups are written ``C`` whatever the check - #: (:func:`disclosure.written_status`), and counted here by the check. + #: histogram's or a quantile's withheld groups are written ``C`` whatever the check, and + #: counted here by the check (:func:`_code`). withheld: Mapping[str, int] = field(default_factory=dict) #: The histogram cells withheld inside published groups, per code of the check #: that withheld them; the document writes each ``C``. @@ -628,8 +632,8 @@ def _declare_statistics( The rows of each statistic are built from the one read of the table. A statistic no record forms a group of (a grouped one over an empty table) has nothing to write, and is returned with the reason. Every statistic is measured - before any is checked, since a quantile's and a grouped histogram's publication - depend on the column's other statistics (:func:`_withheld_beside`). *tallies* + before any is checked, since what a histogram or a quantile may publish depends on + the column's other statistics (:func:`_edge_decisions`). *tallies* count the values and the histogram cells the checks withhold, by the check. """ rows: Dict[Tuple[str, ...], List[Dict[str, Any]]] = {} @@ -657,124 +661,159 @@ def _declare_statistics( left_out.append((statistic.description, "no record forms a group of it")) continue formed.append((statistic, measured)) - beside = _beside(formed, rule) - for statistic, measured in formed: + decided = _edge_decisions(formed, backend, rule) + for position, (statistic, measured) in enumerate(formed): entry = _declared_entry( - statistic, measured, backend, rule, tallies, fractional_bits, beside + statistic, measured, rule, tallies, fractional_bits, decided.get(position) ) columns_doc[statistic.column].setdefault("statistics", []).append(entry) return left_out #: The declared statistics that count records between edges of their column (FLUX ADR -#: 0012): a quantile is published only as the one of them on its column. -_EDGE_COUNTS = frozenset({"marginalHistogram", "levelHistogram", "quantile"}) +#: 0012), in the order a column's call to the output check lists them. +_EDGE_KINDS = ("marginalHistogram", "levelHistogram", "quantile") -@dataclass(frozen=True) -class _Beside: - """What one statistic's groups give back with the column's others (FLUX ADR 0012). +def _code(reason: str) -> str: + """The ``CL_CONF_STATUS`` code the summary tallies a withheld edge count under. - ``crowded`` holds the columns with more than one statistic that counts at their - edges, where a quantile is withheld. ``small`` holds each (column, groupBy) with a - group the size check withholds, the rows no group holds included, whose - histograms and quantiles are then withheld whole. + ``A`` for a small group or cell and for one the rounding withholds, ``D`` for one + withheld so that another cannot be recovered: off the partition, by differencing, + or beside rows no group holds. """ + return disclosure.SMALL_COUNT if reason in _TALLIED_AS_SMALL else disclosure.SECONDARY + - crowded: FrozenSet[str] - small: FrozenSet[Tuple[str, str]] +_TALLIED_AS_SMALL = frozenset({"small", WITHHELD_ROUNDING}) -def _beside( +def _edge_decisions( formed: Sequence[Tuple[DeclaredStatistic, MeasuredStatistic]], + backend: StatisticsBackend, rule: disclosure.DisclosureRule, -) -> _Beside: - """The columns a quantile shares with another count at edges, and the small groupBys.""" - counts = Counter(statistic.column for statistic, _ in formed if statistic.kind in _EDGE_COUNTS) - small = set() - for statistic, measured in formed: - if statistic.group_by is None: +) -> Dict[int, EdgeDecision]: + """What each histogram and quantile may publish, by its position in *formed*. + + The statistics that count at the edges of one column are checked in one call, in + declaration order (FLUX ADR 0012): a statistic checked on its own is judged alone, so + a quantile would pass beside a histogram and no count would be checked against + another. Where a grouped one has rows no group holds that cannot be read as the block + reads them, the check cannot see what its groups give back, and every group of every + such statistic of the column is withheld instead. + """ + columns: Dict[str, List[int]] = {} + for position, (statistic, _) in enumerate(formed): + if statistic.kind in _EDGE_KINDS: + columns.setdefault(statistic.column, []).append(position) + decided: Dict[int, EdgeDecision] = {} + for column, positions in columns.items(): + if any(formed[each][1].left_out_unread for each in positions): + for each in positions: + decided[each] = _withheld_whole(formed[each][1], "rows no group holds are unread") continue - sizes = [group.k for group in measured.groups] - if measured.left_out is not None: - sizes.append(measured.left_out.k) - if measured.left_out_unread or any( - not disclosure.check_group_count(size, rule).published for size in sizes - ): - small.add((statistic.column, statistic.group_by)) - return _Beside( - frozenset(column for column, count in counts.items() if count > 1), frozenset(small) - ) + statistics = [_edge_statistic(*formed[each], rule) for each in positions] + try: + decisions = backend.check_edge_counts( + statistics, + total=_column_total(*formed[positions[0]]), + minimum_cell=rule.minimum_cell, + zeros_disclosive=rule.zeros_disclosive, + round_count=lambda count: disclosure.round_count(count, rule), + ) + except ValueError as exc: + raise ProfileError( + "profile_edge_counts_refused", + f"the counts of column {column!r} at its edges cannot describe one set of " + f"records: {exc}", + {"written": "nothing", "next": "report this: the output check refused its input"}, + ) from None + decided.update(zip(positions, decisions, strict=True)) + return decided -def _withheld_beside( - statistic: DeclaredStatistic, verdicts: List[disclosure.Verdict], beside: _Beside -) -> List[disclosure.Verdict]: - """A histogram's or a quantile's verdicts once the column's other statistics are seen. - - A quantile counts at its witness, an edge no declaration fixes, so beside another - statistic that counts at its column's edges its ranks less theirs would give a - withheld count back. And a group the size check withholds is the column's count - less its siblings' sizes, so every group of its groupBy's histograms and quantiles - goes with it, as a grouped mean's or support's does - (:func:`disclosure.suppress_secondary`). The groups that passed are withheld with - ``D``, and written ``C``. +def _column_total(statistic: DeclaredStatistic, measured: MeasuredStatistic) -> int: + """The records a column's statistics count: its non-null values. + + An ungrouped statistic counts them in its one group, and a grouped one in its groups + and the rows a null ``groupBy`` key leaves out. + """ + held = sum(group.k for group in measured.groups) + if statistic.group_by is not None and measured.left_out is not None: + held += measured.left_out.k + return held + + +def _edge_statistic( + statistic: DeclaredStatistic, measured: MeasuredStatistic, rule: disclosure.DisclosureRule +) -> EdgeStatistic: + """One histogram or quantile as the output check reads it, unrounded. + + What the profiler's own rounding check already withholds (a positive count that + rounds to zero) is passed as withheld; the check decides every other withholding. """ - reason = None - if statistic.kind == "quantile" and statistic.column in beside.crowded: - reason = "another statistic counts at its column's edges" - elif statistic.kind in _EDGE_COUNTS and statistic.group_by is not None: - if (statistic.column, statistic.group_by) in beside.small: - reason = "a group of its groupBy is withheld for its size" - if reason is None: - return verdicts - withheld = disclosure.Verdict(disclosure.SECONDARY, reason) - return [withheld if verdict.published else verdict for verdict in verdicts] + group_status = { + group.key: WITHHELD_ROUNDING + for group in measured.groups + if not disclosure.check_rounding(group.k, rule).published + } + cell_status: Dict[str, Tuple[str, ...]] = {} + if statistic.kind != "quantile": + for group in measured.groups: + statuses = tuple( + ( + disclosure.FREE + if disclosure.check_rounding(count, rule).published + else WITHHELD_ROUNDING + ) + for count in group.cells + ) + if any(status != disclosure.FREE for status in statuses): + cell_status[group.key] = statuses + return EdgeStatistic( + identifier=statistic.identifier, + edges=statistic.edges, + levels=tuple(statistic.shape.get("levels", ())), + groups=tuple(group.statistic for group in measured.groups), + group_by=statistic.group_by, + level=Fraction(statistic.shape["level"]) if statistic.kind == "quantile" else None, + left_out=measured.left_out.statistic if measured.left_out is not None else None, + group_status=group_status, + cell_status=cell_status, + ) + + +def _withheld_whole(measured: MeasuredStatistic, reason: str) -> EdgeDecision: + """Every group withheld, for a reason the output check was not asked about.""" + keys = [group.key for group in measured.groups] + return EdgeDecision( + group_status={key: disclosure.WITHHELD for key in keys}, + cell_status={}, + group_reason={key: reason for key in keys}, + cell_reason={}, + rounded={}, + ) def _declared_entry( statistic: DeclaredStatistic, measured: MeasuredStatistic, - backend: StatisticsBackend, rule: disclosure.DisclosureRule, tallies: Tuple["Counter[str]", "Counter[str]"], fractional_bits: int, - beside: _Beside, + decision: Optional[EdgeDecision], ) -> Dict[str, Any]: """One declared statistic as the document writes it: every group checked, none guessed at. - Each status is written through :func:`disclosure.written_status`, so a histogram's - and a quantile's withheld groups and cells read ``C``; *tallies* count them by the - check. + A histogram's or a quantile's groups come from the *decision* of the output check on + its column, and a mean's and a support's from their own checks. *tallies* count the + groups and the histogram cells that are withheld, by the check. """ - tally, cell_tally = tallies groups = measured.groups - checked = [_check_group(statistic, group, backend, rule, fractional_bits) for group in groups] - verdicts = [verdict for verdict, _ in checked] - if statistic.group_by is not None and statistic.kind in _SECONDARY_SUPPRESSION: - verdicts = _suppressed(statistic, verdicts, measured, backend, rule, fractional_bits) - verdicts = _withheld_beside(statistic, verdicts, beside) - written: List[Dict[str, Any]] = [] - for group, verdict, (_, published) in zip(groups, verdicts, checked, strict=True): - tally[verdict.status] += 1 - if verdict.published: - if "cellStatus" in published: - checks = published["cellStatus"] - cell_tally.update(code for code in checks if code != disclosure.FREE) - published = { - **published, - "cellStatus": [ - disclosure.written_status(code, statistic.kind) for code in checks - ], - } - written.append({**published, "status": disclosure.FREE}) - else: - written.append( - { - "group": group.key, - "status": disclosure.written_status(verdict.status, statistic.kind), - } - ) + if decision is not None: + written = _edge_checked_groups(statistic, measured, rule, tallies, decision) + else: + written = _own_checked_groups(statistic, measured, rule, tallies[0], fractional_bits) entry: Dict[str, Any] = { "definition": statistic.identifier, "sampleSize": disclosure.sample_size(sum(group.k for group in groups), rule), @@ -787,16 +826,117 @@ def _declared_entry( return entry -#: The declared statistics whose groups partition a total the profile publishes, so one -#: withheld group would be given back by the others (:func:`disclosure.suppress_secondary`). -_SECONDARY_SUPPRESSION = frozenset({"mean", "support"}) +def _edge_checked_groups( + statistic: DeclaredStatistic, + measured: MeasuredStatistic, + rule: disclosure.DisclosureRule, + tallies: Tuple["Counter[str]", "Counter[str]"], + decision: EdgeDecision, +) -> List[Dict[str, Any]]: + """A histogram's or a quantile's groups, as the output check on its column decided them.""" + tally, cell_tally = tallies + written: List[Dict[str, Any]] = [] + for group in measured.groups: + if decision.group_status[group.key] == disclosure.FREE and not _every_cell_withheld( + statistic, decision, group.key + ): + tally[disclosure.FREE] += 1 + published = _published_edge_group(statistic, group, decision, rule, cell_tally) + written.append({**published, "status": disclosure.FREE}) + else: + tally[_withheld_code(statistic, decision, group.key)] += 1 + written.append({"group": group.key, "status": disclosure.WITHHELD}) + return written + + +def _every_cell_withheld(statistic: DeclaredStatistic, decision: EdgeDecision, key: str) -> bool: + """Whether a histogram group the check lets through holds no published cell. + + Such a group says nothing but its size, so it is withheld whole, as a group whose + every cell is withheld always was. A quantile's group has no cells. + """ + if statistic.kind == "quantile": + return False + return all(status != disclosure.FREE for status in decision.cell_status[key]) + + +def _withheld_code(statistic: DeclaredStatistic, decision: EdgeDecision, key: str) -> str: + """The code a withheld edge group is tallied under: its own reason, else its cells'. + + A group withheld whole because every cell is withheld counts as small (``A``) where + one of its cells was withheld for its own size, and as secondary (``D``) otherwise. + """ + if decision.group_status[key] != disclosure.FREE: + return _code(decision.group_reason[key]) + codes = { + _code(reason) + for reason, status in zip(decision.cell_reason[key], decision.cell_status[key], strict=True) + if status != disclosure.FREE + } + return disclosure.SMALL_COUNT if disclosure.SMALL_COUNT in codes else disclosure.SECONDARY + + +def _own_checked_groups( + statistic: DeclaredStatistic, + measured: MeasuredStatistic, + rule: disclosure.DisclosureRule, + tally: "Counter[str]", + fractional_bits: int, +) -> List[Dict[str, Any]]: + """A mean's or a support's groups, each by its own check, siblings withheld with a small one.""" + groups = measured.groups + checked = [_check_group(statistic, group, rule, fractional_bits) for group in groups] + verdicts = [verdict for verdict, _ in checked] + if statistic.group_by is not None: + verdicts = _suppressed(statistic, verdicts, measured, rule, fractional_bits) + written: List[Dict[str, Any]] = [] + for group, verdict, (_, published) in zip(groups, verdicts, checked, strict=True): + tally[verdict.status] += 1 + if verdict.published: + written.append({**published, "status": disclosure.FREE}) + else: + written.append({"group": group.key, "status": verdict.status}) + return written + + +def _published_edge_group( + statistic: DeclaredStatistic, + group: MeasuredGroup, + decision: EdgeDecision, + rule: disclosure.DisclosureRule, + cell_tally: "Counter[str]", +) -> Dict[str, Any]: + """A published group of a histogram or a quantile, its counts rounded. + + A quantile's group is spelled by the registry with its counts already rounded. A + histogram's cells are rounded here, and a cell the check withholds is ``null`` with + status ``C`` in ``cellStatus``, which is written only when some cell is withheld. + """ + if statistic.kind == "quantile": + return dict(decision.rounded[group.key]) + published = dict(group.document) + published["k"] = disclosure.round_count(group.k, rule) + statuses = decision.cell_status[group.key] + reasons = decision.cell_reason[group.key] + field_name = "binCounts" if statistic.kind == "marginalHistogram" else "levelCounts" + published[field_name] = [ + disclosure.round_count(count, rule) if status == disclosure.FREE else None + for count, status in zip(group.cells, statuses, strict=True) + ] + if any(status != disclosure.FREE for status in statuses): + published["cellStatus"] = list(statuses) + cell_tally.update( + _code(reason) + for reason, status in zip(reasons, statuses, strict=True) + if status != disclosure.FREE + ) + return published def _suppressed( statistic: DeclaredStatistic, verdicts: List[disclosure.Verdict], measured: MeasuredStatistic, - backend: StatisticsBackend, rule: disclosure.DisclosureRule, fractional_bits: int, ) -> List[disclosure.Verdict]: @@ -811,23 +951,20 @@ def _suppressed( if measured.left_out_unread: hidden.append(disclosure.Verdict(disclosure.SMALL_COUNT, "rows no group holds")) elif measured.left_out is not None: - hidden.append(_check_group(statistic, measured.left_out, backend, rule, fractional_bits)[0]) + hidden.append(_check_group(statistic, measured.left_out, rule, fractional_bits)[0]) return list(disclosure.suppress_secondary([*verdicts, *hidden]))[: len(verdicts)] def _check_group( statistic: DeclaredStatistic, group: MeasuredGroup, - backend: StatisticsBackend, rule: disclosure.DisclosureRule, fractional_bits: int, ) -> Tuple[disclosure.Verdict, Dict[str, Any]]: - """One group's verdict, and its document as it is published when the verdict allows. + """A mean's or a support's group: its verdict, and its document as published when it allows. Every group first passes as a count of records (``k``), which is published rounded; then what its statistic adds is checked as that statistic's rule says. - A histogram's ``cellStatus`` still names the check that withheld each cell: - :func:`_declared_entry` writes it. """ kind = statistic.kind published = dict(group.document) @@ -837,28 +974,8 @@ def _check_group( verdict = disclosure.check_group_count(group.k, rule) if not verdict.published: return verdict, published - if kind in ("marginalHistogram", "levelHistogram"): - cells = disclosure.publish_cells(group.cells, rule) - field_name = "binCounts" if kind == "marginalHistogram" else "levelCounts" - published[field_name] = list(cells.counts) - if cells.any_withheld: - published["cellStatus"] = list(cells.statuses) - return cells.verdict, published if kind == "support": return disclosure.check_support(group.member_holders, rule), published - if kind == "quantile" and group.quantile is not None: - level = Fraction(statistic.shape["level"]) - _, below, at_or_below = group.quantile - verdict, shown_below, shown_at = disclosure.check_quantile( - group.k, - below, - at_or_below, - rule, - lambda k, lower, upper: backend.quantile_ranks_ok(k, level, lower, upper), - ) - published["quantileBelow"] = shown_below - published["quantileAtOrBelow"] = shown_at - return verdict, published raise AssertionError(f"no disclosure rule for declared statistic {statistic.identifier!r}") @@ -873,11 +990,12 @@ def _row_count(rows: int, rule: disclosure.DisclosureRule, tally: "Counter[str]" def _shape(statistic: Mapping[str, Any]) -> Optional[str]: """What tells two statistics of one column and definition apart in the report.""" + parts = [] if "level" in statistic: - return f"at level {statistic['level']}" + parts.append(f"at level {statistic['level']}") if "bins" in statistic: - return f"over bins {', '.join(statistic['bins'])}" - return None + parts.append(f"over bins {', '.join(statistic['bins'])}") + return " ".join(parts) or None def _statistic_summaries( diff --git a/pyproject.toml b/pyproject.toml index 0bd79d18..a5b73af3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -404,16 +404,19 @@ lineage = [ # does. It is standard-library only and supports Python 3.10, so it needs no # marker. # -# flux-stats comes from PyPI. 0.6.2 is the first release that holds both the -# record reader the profiler uses and the fidelity-block grammar --bundle reads -# (flux_stats.fidelity); PyPI's 0.0.1 is a placeholder with no code. It is -# versioned 0.x, where a minor release may change the names +# flux-stats comes from PyPI. The profiler needs a release that holds the record +# reader, the fidelity-block grammar (flux_stats.fidelity), quantile/2, the output +# check for edge counts (flux_stats.edge_counts) and the exact-YAML reader and +# canonical form --bundle reads with (flux_stats.exact_yaml, flux_stats.canonical; +# the ``yaml`` extra brings PyYAML). PyPI's 0.0.1 is a placeholder with no code. It +# is versioned 0.x, where a minor release may change the names # fluid_build/profiler/_stats.py imports, so the range stops at the next minor. # That module repeats the requirement in its install message, and # tests/profiler/test_flux_stats_pin.py keeps the two equal. +# 0.7.0 is the first flux-stats release that holds these modules. profile = [ "duckdb>=1.5.0", - "flux-stats>=0.6.2,<0.7", + "flux-stats[yaml]>=0.7.0,<0.8", ] # AI + MCP together: restores the pre-0.28.0 base behaviour (the diff --git a/tests/profiler/_fake_backend.py b/tests/profiler/_fake_backend.py index 9358bcb5..9f73e8ff 100644 --- a/tests/profiler/_fake_backend.py +++ b/tests/profiler/_fake_backend.py @@ -29,7 +29,7 @@ from pathlib import Path from typing import Any, Callable, Dict, FrozenSet, List, Sequence, Tuple -from fluid_build.profiler._stats import DocumentRefused, ReadRefused +from fluid_build.profiler._stats import BundleTextRefused, DocumentRefused, ReadRefused FIXTURES = Path(__file__).parent / "fixtures" SCHEMA_PATH = FIXTURES / "profile-1.schema.json" @@ -212,3 +212,19 @@ def check_document(self, text: str) -> None: ) if errors: raise DocumentRefused("; ".join(error.message for error in errors[:3])) + + def read_budget(self) -> object: + return object() + + def read_documents(self, text: str, *, name: str, budget: object) -> Tuple[Dict[str, Any], ...]: + """Plain YAML: enough for a bundle's file listing, which the real reader tests cover.""" + import yaml + + try: + documents = list(yaml.safe_load_all(text)) + except yaml.YAMLError: + raise BundleTextRefused("FLX-3105", f"{name}: unreadable YAML") from None + return tuple(document for document in documents if isinstance(document, dict)) + + def canonicalise(self, value: Any, *, name: str) -> Any: + return value diff --git a/tests/profiler/test_bundle.py b/tests/profiler/test_bundle.py index 09499768..8bd7c7d9 100644 --- a/tests/profiler/test_bundle.py +++ b/tests/profiler/test_bundle.py @@ -7,8 +7,10 @@ import pytest -from fluid_build.profiler import bundle -from fluid_build.profiler.bundle import BundleError, read_bundle_block +from fluid_build.profiler import _stats, bundle +from fluid_build.profiler.bundle import BundleError + +flux_stats = pytest.importorskip("flux_stats") pytestmark = pytest.mark.unit @@ -28,6 +30,11 @@ """ +def read_bundle_block(directory, expose_id): + """The bundle's block, read through flux-stats' exact-YAML reader as the CLI reads it.""" + return bundle.read_bundle_block(directory, expose_id, _stats.load_backend()) + + def _write(directory, name, text): (directory / name).write_text(text, encoding="utf-8") @@ -108,7 +115,7 @@ def test_an_unreadable_document_names_the_file_and_the_line_and_quotes_nothing(t with pytest.raises(BundleError) as refused: read_bundle_block(tmp_path, "orders") assert refused.value.event == "profile_bundle_unreadable" - assert "simulation.flux.yml" in str(refused.value) and "line" in str(refused.value) + assert "simulation.flux.yml" in str(refused.value) and "unreadable YAML" in str(refused.value) assert "value-77" not in str(refused.value) @@ -192,10 +199,10 @@ def test_a_bare_float_key_is_its_decimal_spelling_as_the_engine_reads_it(tmp_pat def test_a_binary_float_key_is_refused_and_names_the_file(): # The loader never builds a float, so this is the canonical form's own guard. - with pytest.raises(BundleError) as refused: - bundle._canonicalise({0.5: "x"}, "s.flux.yml", "fidelity") - assert refused.value.event == "profile_bundle_unreadable" - assert "s.flux.yml" in str(refused.value) and "float key" in str(refused.value) + with pytest.raises(_stats.BundleTextRefused) as refused: + _stats.load_backend().canonicalise({0.5: "x"}, name="s.flux.yml: fidelity") + assert "s.flux.yml" in str(refused.value) and "float" in str(refused.value) + assert refused.value.code == "FLX-1001" def test_an_integer_outside_the_safe_range_is_refused(tmp_path): @@ -221,21 +228,21 @@ def test_a_block_whose_aliases_write_out_too_many_values_is_refused(tmp_path): with pytest.raises(BundleError) as refused: read_bundle_block(tmp_path, "orders") assert refused.value.event == "profile_bundle_unreadable" - assert "more than 100000 values" in str(refused.value) + assert "more than 1000000 values" in str(refused.value) def test_a_block_that_holds_itself_through_an_alias_is_refused(tmp_path): _write(tmp_path, "s.flux.yml", _simulation_with("r: &r [*r]")) with pytest.raises(BundleError) as refused: read_bundle_block(tmp_path, "orders") - assert "nests deeper than 64 levels" in str(refused.value) + assert "holds itself through an alias" in str(refused.value) def test_a_number_too_long_to_write_out_is_refused(tmp_path): _write(tmp_path, "s.flux.yml", _simulation_with("columns: {amount: {max: 1.0e+300000000}}")) with pytest.raises(BundleError) as refused: read_bundle_block(tmp_path, "orders") - assert "more than 1000 digits" in str(refused.value) + assert "more than 1000 digits" in str(refused.value) and "line" in str(refused.value) def test_a_document_nested_too_deeply_to_read_is_refused(tmp_path): diff --git a/tests/profiler/test_bundle_flux_stats.py b/tests/profiler/test_bundle_flux_stats.py index f90a52eb..ac44fb4b 100644 --- a/tests/profiler/test_bundle_flux_stats.py +++ b/tests/profiler/test_bundle_flux_stats.py @@ -34,7 +34,7 @@ from flux_stats.profile import load_profile_text # noqa: E402 from fluid_build.cli import main # noqa: E402 -from fluid_build.profiler import _stats, bundle, disclosure, runner # noqa: E402 +from fluid_build.profiler import _stats, bundle, disclosure, document # noqa: E402 from fluid_build.profiler._stats import DeclaredStatistic # noqa: E402 from fluid_build.profiler.runner import ProfileError, profile_expose # noqa: E402 from tests.profiler.test_flux_stats_backend import _slice # noqa: E402 @@ -147,7 +147,7 @@ def _profile( ): chosen = properties if properties is not None else PROPERTIES directory = _bundle(tmp_path, chosen, bits, market_null) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) return profile_expose( CONTRACT, EXPOSE, @@ -215,11 +215,13 @@ def test_the_document_answers_every_declared_property(tmp_path): levels = _entry(result, "market", "levelHistogram/1") assert levels["levels"] == ["DE", "FR", "GB", "IE"] (level_group,) = levels["groups"] - # DE is declared and no record holds it: a zero cell is withheld, the rest are published. - # A histogram's withheld cell is written C, whatever withheld it; the result keeps why. - assert level_group["levelCounts"] == [None, 40, 40, 40] - assert level_group["cellStatus"] == ["C", "F", "F", "F"] - assert result.withheld_cells == {"A": 1} + # DE is declared and no record holds it: a zero cell is withheld, and with the column's + # count known, the three others would give it back, so FR is withheld with it (a formed + # zero is protected). A histogram's withheld cell is written C, whatever withheld it; + # the result keeps why: DE is small, FR is secondary. + assert level_group["levelCounts"] == [None, None, 40, 40] + assert level_group["cellStatus"] == ["C", "C", "F", "F"] + assert result.withheld_cells == {"A": 1, "D": 1} assert _entry(result, "market", "support/1")["groups"][0]["support"] == ["FR", "GB", "IE"] @@ -400,29 +402,200 @@ def test_a_group_withheld_for_its_size_withholds_its_siblings_in_a_grouped_quant assert result.withheld["D"] == 3 -@pytest.mark.parametrize("kind", ["marginalHistogram", "levelHistogram", "quantile"]) -def test_the_sibling_rule_reaches_every_histogram_and_quantile_of_the_column(kind): - # A level histogram needs a second categorical column to be grouped by, which this - # module's contract lacks, so the rule is checked on its own. - passed = disclosure.PUBLISHED - beside = runner._Beside(crowded=frozenset(), small=frozenset({("market", "tier")})) - - def declared(kind: str, column: str) -> DeclaredStatistic: - return DeclaredStatistic( - identifier=f"{kind}/1", - kind=kind, - column=column, - group_by="tier", - shape={}, - columns=(column, "tier"), - read_as={}, - ) +@pytest.mark.parametrize( + "grouped", + [ + pytest.param({**PROPERTIES[2], "groupBy": "market"}, id="marginalHistogram"), + pytest.param( + {**PROPERTIES[5], "groupBy": "market", "bins": ["0", "50", "100"]}, id="quantile-2" + ), + ], +) +def test_a_small_group_withholds_every_group_of_its_groupby_in_each_statistic(tmp_path, grouped): + rows = _rows() + for row in rows[:3]: + row["market"] = "DE" # a group of three records + result = _profile(tmp_path, rows, [grouped]) + (entry,) = [e for e in _entries(result, "amount") if e.get("groupBy") == "market"] + assert {g["status"] for g in entry["groups"]} == {"C"} + assert all(set(g) == {"group", "status"} for g in entry["groups"]) + + +def _row(amount: str, market: str = "IE") -> Dict[str, Any]: + return {"amount": amount, "market": market, "qty": "1"} + + +def _histogram(bins: List[str], column: str = "amount", **more: Any) -> Dict[str, Any]: + return { + "property": "marginalHistogram", + "column": column, + "bins": bins, + "tolerance": _DISTANCE, + **more, + } + + +def _histograms(result, column: str = "amount") -> List[Dict[str, Any]]: + return [e for e in _entries(result, column) if e["definition"] == "marginalHistogram/1"] + + +def test_a_histogram_and_a_coarser_one_of_a_column_give_no_withheld_cell_back(tmp_path): + # Fine cells of 30, 5, 40 and 45 records: the 5 is small, and the coarse cell of 35 beside + # the 30 would give it back by subtraction. Neither histogram is checked alone. + rows = [_row("10")] * 30 + [_row("30")] * 5 + [_row("60")] * 40 + [_row("80")] * 45 + fine = _histogram(["0", "25", "50", "75", "100"]) + coarse = _histogram(["0", "50", "100"]) + result = _profile(tmp_path, rows, [fine, coarse]) + fine_group, coarse_group = (histogram["groups"][0] for histogram in _histograms(result)) + # A group whose every cell is withheld is withheld whole, with no cells written. + fine_cells = fine_group.get("binCounts", [None] * 4) + coarse_cells = coarse_group.get("binCounts", [None] * 2) + assert fine_cells[1] is None # the 5 itself + for cell, inside in ((0, fine_cells[:2]), (1, fine_cells[2:])): + if coarse_cells[cell] is not None: + # A published coarse cell with exactly one withheld cell inside it gives that cell back. + assert sum(count is None for count in inside) != 1 + published = [count for count in fine_cells + coarse_cells if count is not None] + assert 5 not in published and 35 not in published + withheld = [count is None for count in fine_cells] + assert sum(withheld) >= 2 # a single withheld cell is the total less the published ones + assert result.withheld_cells["D"] >= 1 # more than the small cell itself was withheld + load_profile_text(result.canonical.decode("utf-8")) + + +def test_a_level_histogram_with_one_small_level_withholds_a_neighbour(tmp_path): + # Levels DE, FR, GB, IE hold 5, 25, 40 and 50 records. The 5 is small, and the column's + # count less the three published levels is exactly that 5: another level goes with it. + rows = ( + [_row("10", "IE")] * 50 + + [_row("10", "GB")] * 40 + + [_row("10", "FR")] * 25 + + [_row("10", "DE")] * 5 + ) + result = _profile(tmp_path, rows, [PROPERTIES[3]]) + (group,) = _entry(result, "market", "levelHistogram/1")["groups"] + assert group["cellStatus"][0] == "C" # DE, the small level + assert group["cellStatus"].count("C") == 2 + assert group["levelCounts"][0] is None and None in group["levelCounts"][1:] + assert result.withheld_cells == {"A": 1, "D": 1} + + +def test_a_formed_zero_is_protected_under_the_default_rule(tmp_path): + # Cells of 12, 0 and 12 records: the zero is withheld (ACRO's default holds zeros + # disclosive), and the column's 24 less the two published 12s would give it back. + rows = [_row("5")] * 12 + [_row("25")] * 12 + result = _profile(tmp_path, rows, [_histogram(["0", "10", "20", "100"])]) + (group,) = _histograms(result)[0]["groups"] + assert group["cellStatus"][1] == "C" and group["binCounts"][1] is None + assert group["cellStatus"].count("C") == 2 + assert group["binCounts"][0] == 10 or group["binCounts"][2] == 10 # the 12 left, rounded + assert result.withheld_cells == {"A": 1, "D": 1} + + +def test_a_formed_zero_is_published_where_the_rule_holds_zeros_safe(tmp_path): + from dataclasses import replace + + rule = replace(disclosure.ACRO_DEFAULT, name="zeros-safe", zeros_disclosive=False) + directory = _bundle(tmp_path, [_histogram(["0", "10", "20", "100"])]) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) + rows = [_row("5")] * 12 + [_row("25")] * 12 + result = profile_expose( + CONTRACT, + EXPOSE, + source=_slice(tmp_path, rows), + backend=_stats.load_backend(), + cwd=tmp_path, + bundle=block, + rule=rule, + ) + (group,) = _histograms(result)[0]["groups"] + assert group["binCounts"] == [10, 0, 10] and "cellStatus" not in group + + +def test_a_quantile_with_bins_is_written_as_quantile_2(tmp_path): + binned = {**PROPERTIES[5], "bins": ["0", "50", "100"]} + result = _profile(tmp_path, _rows(), [binned]) + entry = _entry(result, "amount", "quantile/2") + assert entry["level"] == "0.5" and entry["bins"] == ["0", "50", "100"] + # 120 records, the median is the 60th, a 40: in the first bin, with 72 records through it + # (rounded to 70), none below it, and 48 above it. + assert entry["groups"] == [ + {"group": "", "k": 120, "quantileBin": 0, "binBelow": 0, "binThrough": 70, "status": "F"} + ] + assert "quantileWitness" not in json.dumps(entry) + assert result.document["_schemaURL"] == document.SCHEMA_URL_1_1_0 + profile, _ = load_profile_text(result.canonical.decode("utf-8")) + (read,) = [ + e for e in profile.column_statistics(EXPOSE, "amount") if e.identifier == "quantile/2" + ] + assert read.bins == (Fraction(0), Fraction(50), Fraction(100)) + assert read.level == Fraction(1, 2) + (group,) = read.groups + assert group.published and group.statistic.quantile_bin == 0 + assert (group.statistic.bin_below, group.statistic.bin_through) == (0, 70) + assert result.statistics.count( + ("amount", "quantile/2", None, "at level 0.5 over bins 0, 50, 100") + ) + + +def test_the_engines_matcher_finds_a_quantile_with_bins(tmp_path): + binned = {**PROPERTIES[5], "bins": ["0", "50", "100"]} + result = _profile(tmp_path, _rows(), [binned]) + profile, _ = load_profile_text(result.canonical.decode("utf-8")) + expose = expose_from_contract(CONTRACT, EXPOSE) + parsed = parse_fidelity_block(_block([binned]), expose=expose) + from flux_stats.fidelity import definition_for_property + + (declaration,) = parsed.properties + found = [ + entry + for entry in profile.column_statistics(EXPOSE, "amount") + if entry.identifier == definition_for_property(declaration).identifier + and entry.bins == declaration.bins + and entry.level == declaration.level + ] + assert len(found) == 1 + + +def test_a_profile_without_a_quantile_with_bins_keeps_format_1_0_0(tmp_path): + result = _profile(tmp_path, _rows()) + assert result.document["_schemaURL"] == document.SCHEMA_URL + assert "quantile/2" not in result.canonical.decode("utf-8") + - withheld = runner._withheld_beside(declared(kind, "market"), [passed, passed], beside) - assert [verdict.status for verdict in withheld] == ["D", "D"] - # The same groupBy on another column, and a mean, are left to their own checks. - assert runner._withheld_beside(declared(kind, "channel"), [passed], beside) == [passed] - assert runner._withheld_beside(declared("mean", "market"), [passed], beside) == [passed] +def test_a_quantile_with_bins_beside_a_histogram_on_its_partition_is_published(tmp_path): + # The bins of the quantile are the histogram's: one partition, so it is not withheld + # for standing beside another count at the column's edges, as a quantile/1 is. + binned = {**PROPERTIES[5], "bins": ["0", "50", "100"]} + result = _profile(tmp_path, _rows(), [PROPERTIES[2], binned]) + (group,) = _entry(result, "amount", "quantile/2")["groups"] + assert group["status"] == "F" + assert _histograms(result)[0]["groups"][0]["status"] == "F" + + +def test_rows_no_group_holds_are_one_more_group_the_check_counts_and_never_writes(tmp_path): + rows = _rows() + for row in rows[:15]: + row["market"] = None # fifteen records in no group, and in the column's count + by_market = {**PROPERTIES[2], "groupBy": "market"} + result = _profile(tmp_path, rows, [by_market], market_null="exclude") + grouped = _entry(result, "amount", "marginalHistogram/1", "market") + assert [g["group"] for g in grouped["groups"]] == ["FR", "GB", "IE"] + assert {g["status"] for g in grouped["groups"]} == {"F"} + assert b"\\u0000" not in result.canonical and b"\x00" not in result.canonical + + +def test_a_refusal_of_the_counts_by_the_output_check_stops_the_profile_naming_no_value( + tmp_path, monkeypatch +): + def refuse(self, statistics, **_kwargs): + raise ValueError("the groups of 'market' do not add up to the column's total") + + monkeypatch.setattr(_stats._FluxStatsBackend, "check_edge_counts", refuse) + with pytest.raises(ProfileError) as refused: + _profile(tmp_path, _rows(), [PROPERTIES[2]]) + assert refused.value.event == "profile_edge_counts_refused" + assert "column 'amount'" in str(refused.value) def test_a_quantile_beside_a_histogram_of_another_column_is_published(tmp_path): @@ -434,6 +607,8 @@ def test_a_quantile_beside_a_histogram_of_another_column_is_published(tmp_path): def test_a_small_group_of_one_column_leaves_another_columns_groups_published(tmp_path): rows = _rows() + for index, row in enumerate(rows): + row["qty"] = "60" if index % 2 else "10" # each group of qty fills both bins for index, row in enumerate(rows[:15]): row["market"] = "DE" if index < 12: @@ -446,7 +621,10 @@ def test_a_small_group_of_one_column_leaves_another_columns_groups_published(tmp amount = _entry(result, "amount", "marginalHistogram/1", "market") assert {g["status"] for g in amount["groups"]} == {"C"} qty = _entry(result, "qty", "marginalHistogram/1", "market") - assert {g["status"] for g in qty["groups"]} == {"F"} + statuses = {g["group"]: g["status"] for g in qty["groups"]} + # DE's 15 records split 8 and 7, so both its cells are small and it is withheld whole; + # the other markets' groups of qty are published, whatever amount's small group did. + assert statuses == {"DE": "C", "FR": "F", "GB": "F", "IE": "F"} def test_rows_no_group_holds_that_the_block_refuses_withhold_a_grouped_histogram(tmp_path): @@ -538,7 +716,7 @@ def test_two_quantiles_of_one_column_are_named_apart_by_level(tmp_path): assert shapes == {"at level 0.5", "at level 0.25"} -def test_a_zero_cell_is_withheld_and_the_group_published(tmp_path): +def test_a_zero_cell_is_withheld_with_the_cell_that_gives_it_back(tmp_path): rows = [{"amount": "10", "market": "IE", "qty": "1"} for _ in range(120)] properties = [ { @@ -550,8 +728,11 @@ def test_a_zero_cell_is_withheld_and_the_group_published(tmp_path): ] result = _profile(tmp_path, rows, properties) (group,) = _entry(result, "amount", "marginalHistogram/1")["groups"] - assert group["binCounts"] == [None, 120] and group["cellStatus"] == ["C", "F"] - assert result.withheld_cells == {"A": 1} + # The zero cell is withheld, and the 120 beside it would give it back: it goes too, + # and a group with no cell left says nothing, so it is withheld whole, as small. + assert group == {"group": "", "status": "C"} + assert result.withheld_cells == {} # no cell of it is written + assert result.withheld["A"] >= 1 # the group, among the profile's withheld values def test_a_small_table_withholds_the_histogram_whole(tmp_path): @@ -628,7 +809,7 @@ def test_a_value_outside_the_declared_range_is_refused(tmp_path): def test_a_block_the_grammar_refuses_is_refused_before_any_record_is_read(tmp_path): properties = [{"property": "mean", "column": "nope", "tolerance": _ABSOLUTE}] directory = _bundle(tmp_path, properties) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) with pytest.raises(ProfileError) as refused: profile_expose( CONTRACT, @@ -645,7 +826,7 @@ def test_a_declaration_over_the_wrong_domain_is_refused(tmp_path): contract = json.loads(json.dumps(CONTRACT)) contract["exposes"][0]["contract"]["schema"][1]["type"] = "integer" # market as a number directory = _bundle(tmp_path, [PROPERTIES[4]]) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) with pytest.raises(ProfileError) as refused: profile_expose( contract, @@ -662,7 +843,7 @@ def test_a_declared_column_the_contract_types_unreadably_is_refused(tmp_path): contract = json.loads(json.dumps(CONTRACT)) contract["exposes"][0]["contract"]["schema"][0]["type"] = "double" directory = _bundle(tmp_path, [PROPERTIES[0]]) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) with pytest.raises(ProfileError) as refused: profile_expose( contract, @@ -677,21 +858,13 @@ def test_a_declared_column_the_contract_types_unreadably_is_refused(tmp_path): def test_the_seam_deduplicates_identical_properties(tmp_path): directory = _bundle(tmp_path, [PROPERTIES[0], PROPERTIES[0], PROPERTIES[6]]) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) declared = _stats.load_backend().read_block(block.raw, CONTRACT, EXPOSE) assert [s.description for s in declared.statistics] == ["mean/1 of 'amount'"] assert declared.columns == ("amount",) assert [name for name, _ in declared.left_out] == ["correlation of 'amount' and 'qty'"] -def test_quantile_ranks_are_proved_by_the_registry(tmp_path): - backend = _stats.load_backend() - # 120 records, the median is the 60th: 48 strictly below, 72 at or below proves it. - assert backend.quantile_ranks_ok(120, Fraction(1, 2), 48, 72) - assert not backend.quantile_ranks_ok(120, Fraction(1, 2), 60, 70) - assert not backend.quantile_ranks_ok(120, Fraction(1, 2), 70, 45) # not a sub-count - - def test_the_command_writes_the_document_and_names_what_it_answers(tmp_path, monkeypatch, capsys): directory = _bundle(tmp_path, PROPERTIES) monkeypatch.chdir(tmp_path) @@ -714,8 +887,12 @@ def test_the_command_writes_the_document_and_names_what_it_answers(tmp_path, mon assert "levelHistogram/1 of 'market'" in out and "quantile/1 of 'amount'" in out assert "Not answered: correlation of 'amount' and 'qty':" in out assert "a run that checks it is refused (FLX-5037)." in out - # The DE level holds no record: its cell is withheld, written C, and counted by its check. - assert "Withheld histogram cells, by the check (CL_CONF_STATUS code): A (small count) 1." in out + # The DE level holds no record: its cell is withheld, written C, and counted by its check, + # and FR goes with it so that the count cannot be given back. + assert ( + "Withheld histogram cells, by the check (CL_CONF_STATUS code): " + "A (small count) 1, D (secondary suppression) 1." in out + ) assert _ONE_CODE in out written = (tmp_path / "payment.profile.json").read_bytes() load_profile_text(written.decode("utf-8")) @@ -777,7 +954,7 @@ def test_a_flux_stats_without_the_grammar_refuses_before_reading(tmp_path, monke assert rc == 1 assert "profile_statistics_unavailable" in out assert "reads the fidelity block with flux-stats, and the installed flux-stats" in out - assert "lacks flux_stats.fidelity. It needs flux-stats>=0.6.2,<0.7" in out + assert "lacks flux_stats.fidelity. It needs flux-stats[yaml]>=0.7.0,<0.8" in out assert "next: pip install 'data-product-forge[profile]'" in out @@ -790,7 +967,7 @@ def test_a_block_written_with_bare_null_keys_and_bare_floats_profiles_as_the_quo text = path.read_text(encoding="utf-8") assert "'null':" in text and "level: '0.5'" in text path.write_text(text.replace("'null':", "null:").replace("level: '0.5'", "level: 0.5")) - block = bundle.read_bundle_block(directory, EXPOSE) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) assert block.raw["columns"]["market"]["null"] == "fail" bare = profile_expose( CONTRACT, @@ -838,32 +1015,32 @@ def test_the_report_names_the_statistics_written_with_every_group_withheld( def _partly_rows() -> List[Dict[str, Any]]: rows = _rows() for row in rows[:12]: - # Twelve records, 8 and 4 in the two bins: every cell is small, so the group is - # withheld, and its size is not, so the other groups are published. + # Twelve records in one group, 8 and 4 in the two bins: the cell that holds the + # median is small, so that group is withheld, and its size is not, so the other + # groups are published. row["market"] = "DE" return rows _PARTLY_PROPERTIES = [ { - "property": "marginalHistogram", + "property": "quantile", "column": "amount", "groupBy": "market", + "level": "0.5", "bins": ["0", "50", "100"], - "tolerance": _DISTANCE, + "tolerance": _ABSOLUTE, } ] def test_a_statistic_with_some_groups_withheld_is_named_with_its_counts(tmp_path): result = _profile(tmp_path, _partly_rows(), _PARTLY_PROPERTIES) - statuses = [ - g["status"] for g in _entry(result, "amount", "marginalHistogram/1", "market")["groups"] - ] + statuses = [g["status"] for g in _entry(result, "amount", "quantile/2", "market")["groups"]] published = statuses.count("F") assert 0 < published < len(statuses) assert set(statuses) == {"F", "C"} - named = ("amount", "marginalHistogram/1", "market", "over bins 0, 50, 100") + named = ("amount", "quantile/2", "market", "at level 0.5 over bins 0, 50, 100") assert result.partly_withheld_statistics == ((*named, published, len(statuses)),) assert named not in result.statistics assert named not in result.withheld_statistics @@ -888,7 +1065,8 @@ def test_the_report_names_the_statistics_withheld_in_part(tmp_path, monkeypatch, assert rc == 0, out assert ( "Partly withheld, so a run that checks them scores only the published groups: " - "marginalHistogram/1 of 'amount' over bins 0, 50, 100 by 'market' (3 of 4 groups)." in out + "quantile/2 of 'amount' at level 0.5 over bins 0, 50, 100 by 'market' " + "(3 of 4 groups)." in out ) assert _ONE_CODE in out and "Withheld histogram cells" not in out @@ -943,3 +1121,69 @@ def test_a_report_with_no_withheld_histogram_value_says_nothing_of_c(tmp_path, m out = " ".join(capsys.readouterr().out.split()) assert rc == 0, out assert _ONE_CODE not in out and "Withheld histogram cells" not in out + + +@pytest.mark.parametrize( + "declared", + [ + { + "property": "marginalHistogram", + "bins": ["0", "0.4", "0.6", "100"], + "tolerance": _DISTANCE, + }, + {"property": "quantile", "level": "0.5", "bins": ["0", "0.4", "0.6", "100"]}, + ], + ids=["histogram", "quantile"], +) +def test_bins_that_are_no_grid_points_are_refused_before_any_record_is_read(tmp_path, declared): + # At fractionalBits 0 the edges 0.4 and 0.6 are no grid points, and both fall to the + # grid point 0: the fidelity check could not bin a record on them (FLX-5004), and the + # output check would have no partition to read. The block is refused as the check + # refuses it, naming the property and the edge, never a value. + prop = {"column": "amount", "tolerance": _ABSOLUTE, **declared} + directory = _bundle(tmp_path, [prop], bits=0) + block = bundle.read_bundle_block(directory, EXPOSE, _stats.load_backend()) + with pytest.raises(ProfileError) as refused: + profile_expose( + CONTRACT, + EXPOSE, + source=str(tmp_path / "missing.ndjson"), # never read + backend=_stats.load_backend(), + cwd=tmp_path, + bundle=block, + ) + assert refused.value.event == "profile_bundle_block_refused" + said = str(refused.value) + assert "FLX-5004" in said and "properties[0]" in said and "bins[1]" in said + assert "0.4" not in said + + +def test_a_histogram_group_with_no_cell_left_is_withheld_whole(tmp_path): + # DE's 12 records fall 8 and 4 in the two bins, so both its cells are small. A group + # with no cell left says nothing but its size, so it is withheld whole, as a group + # whose every cell is withheld always was, and the histogram is named as withheld in + # part, not as answered. + histogram = _histogram(["0", "50", "100"], groupBy="market") + result = _profile(tmp_path, _partly_rows(), [histogram]) + entry = _entry(result, "amount", "marginalHistogram/1", "market") + groups = {group["group"]: group for group in entry["groups"]} + assert groups["DE"] == {"group": "DE", "status": "C"} + assert {key for key, group in groups.items() if group["status"] == "F"} == {"FR", "GB", "IE"} + (named,) = [one for one in result.partly_withheld_statistics if one[1] == "marginalHistogram/1"] + assert named[-2:] == (3, 4) + load_profile_text(result.canonical.decode("utf-8")) + + +def test_a_refusal_inside_the_output_check_names_only_its_code(tmp_path, monkeypatch): + # The registry's words for a refusal may give counts, so the profile says its code. + from flux_stats import edge_counts, fixedpoint + from flux_stats.errors import FluxError + + def refuse(*_args: Any, **_kwargs: Any) -> Any: + raise FluxError(fixedpoint.FLX_5004, "witness ranks 3/7 are not a sub-count of 9") + + monkeypatch.setattr(edge_counts, "check_edge_counts", refuse) + with pytest.raises(ProfileError) as refused: + _profile(tmp_path, _rows(), [_histogram(["0", "50", "100"])]) + said = str(refused.value) + assert "FLX-5004" in said and "3/7" not in said and "9" not in said.split("FLX-5004")[-1] diff --git a/tests/profiler/test_cli_profile.py b/tests/profiler/test_cli_profile.py index 9fe8d4c2..9a88d2c4 100644 --- a/tests/profiler/test_cli_profile.py +++ b/tests/profiler/test_cli_profile.py @@ -102,7 +102,7 @@ def test_without_the_registry_nothing_is_read_and_nothing_written(workspace, cap assert rc == 1 assert "profile_statistics_unavailable" in out assert "and flux-stats is not installed" in out - assert "It needs flux-stats>=0.6.2,<0.7" in out + assert "It needs flux-stats[yaml]>=0.7.0,<0.8" in out assert "next: pip install 'data-product-forge[profile]'" in out assert "git+https" not in out assert not (workspace / "orders.profile.json").exists() diff --git a/tests/profiler/test_disclosure.py b/tests/profiler/test_disclosure.py index 7582fdf9..528a5478 100644 --- a/tests/profiler/test_disclosure.py +++ b/tests/profiler/test_disclosure.py @@ -394,37 +394,6 @@ def test_a_k_that_rounds_to_zero_is_withheld_under_a_low_threshold(): assert disclosure.check_group_count(3, rule).status == "F" -def test_cells_are_checked_one_by_one_and_rounded(): - cells = disclosure.publish_cells([0, 3, 12, 40], ACRO_DEFAULT) - assert cells.statuses == ("A", "A", "F", "F") - assert cells.counts == (None, None, 10, 40) - assert cells.any_withheld - assert cells.verdict.published - - -def test_cells_that_all_pass_carry_no_withheld_cell(): - cells = disclosure.publish_cells([10, 25, 31], ACRO_DEFAULT) - assert cells.statuses == ("F", "F", "F") - assert cells.counts == (10, 25, 30) - assert not cells.any_withheld - - -def test_a_group_whose_every_cell_is_withheld_is_withheld_whole(): - cells = disclosure.publish_cells([0, 4, 9], ACRO_DEFAULT) - assert cells.verdict.status == "A" - assert cells.counts == (None, None, None) - - -def test_a_zero_cell_is_published_where_the_rule_holds_zeros_safe(): - cells = disclosure.publish_cells([0, 20], _rule(zeros_disclosive=False)) - assert cells.statuses == ("F", "F") and cells.counts == (0, 20) - - -def test_a_positive_cell_that_rounds_to_zero_is_withheld(): - cells = disclosure.publish_cells([2, 30], _rule(minimum_cell=1)) - assert cells.statuses == ("A", "F") - - @pytest.mark.parametrize( ("holders", "status"), [((10, 12), "F"), ((10, 9), "A"), ((1,), "A"), ((), "F"), ((40, 0), "A")], @@ -433,118 +402,9 @@ def test_a_support_needs_every_member_held_by_enough_records(holders, status): assert disclosure.check_support(holders, ACRO_DEFAULT).status == status -def _proves(k, below, at_or_below, level=Fraction(1, 2)): - index = -((-level.numerator * k) // level.denominator) - return below < index <= at_or_below - - -def test_a_quantile_witness_needs_enough_holders_and_publishes_rounded_ranks(): - verdict, below, at_or_below = disclosure.check_quantile(120, 48, 72, ACRO_DEFAULT, _proves) - assert verdict.status == "F" and (below, at_or_below) == (50, 70) - verdict, *_ = disclosure.check_quantile(120, 48, 57, ACRO_DEFAULT, _proves) - assert verdict.status == "A" # nine records hold the witness - verdict, *_ = disclosure.check_quantile(120, 48, 58, ACRO_DEFAULT, _proves) - assert verdict.status == "F" # ten do - - -def test_a_quantile_whose_rounded_ranks_no_longer_prove_the_witness_is_withheld(): - # 40 records, the median is the 20th. 17 below and 27 at or below prove it, with - # 10 holders and 13 records above, and so do their rounded 15 and 25. - assert _proves(40, 17, 27) - verdict, below, at_or_below = disclosure.check_quantile(40, 17, 27, ACRO_DEFAULT, _proves) - assert (below, at_or_below) == (15, 25) and verdict.status == "F" - # 18 below and 28 at or below prove it too, and round to 20 and 30: 20 records - # below no longer leaves the 20th record in the witness's place. - assert _proves(40, 18, 28) - verdict, below, at_or_below = disclosure.check_quantile(40, 18, 28, ACRO_DEFAULT, _proves) - assert (below, at_or_below) == (20, 30) - assert verdict.status == "A" and "no longer prove" in verdict.reason - - -@pytest.mark.parametrize( - ("k", "below", "at_or_below", "status", "side"), - [ - # The records below the witness, holding it and above it: each at least - # minimumCell (FLUX ADR 0012's three cells). None is a zero, and ACRO's - # default holds zeros disclosive. - (120, 9, 30, "A", "below"), - (120, 1, 30, "A", "below"), - (120, 10, 30, "F", None), - (120, 0, 30, "A", "below"), # the witness is the smallest value - (120, 50, 111, "A", "above"), - (120, 50, 119, "A", "above"), - (120, 50, 110, "F", None), - (120, 50, 120, "A", "above"), # the witness is the largest value - (10, 0, 10, "A", "below"), # every record holds it - ], -) -def test_a_quantile_needs_enough_records_below_and_above_its_witness( - k, below, at_or_below, status, side -): - verdict, *_ = disclosure.check_quantile(k, below, at_or_below, ACRO_DEFAULT, lambda *_: True) - assert verdict.status == status - if side is not None: - assert f"{side} the value" in verdict.reason - - -@pytest.mark.parametrize(("below", "at_or_below"), [(0, 30), (50, 120)]) -def test_a_quantile_at_an_end_is_published_where_zeros_are_not_disclosive(below, at_or_below): - rule = _rule(zeros_disclosive=False) - verdict, *_ = disclosure.check_quantile(120, below, at_or_below, rule, lambda *_: True) - assert verdict.status == "F" - - -def test_the_outer_cells_are_checked_before_rounding(): - # 3 records above would round to 5 and read as a cell the threshold cleared. - verdict, *_ = disclosure.check_quantile(60, 20, 57, ACRO_DEFAULT, lambda *_: True) - assert verdict.status == "A" and "above" in verdict.reason - - -@pytest.mark.parametrize( - ("k", "below", "at_or_below"), [(120, 50, 40), (120, 50, 121), (120, -1, 30)] -) -def test_quantile_ranks_that_cannot_describe_the_group_are_refused(k, below, at_or_below): - with pytest.raises(ValueError): - disclosure.check_quantile(k, below, at_or_below, ACRO_DEFAULT, lambda *_: True) - - -def test_a_quantile_rank_that_rounds_to_zero_withholds_the_group(): - rule = _rule(minimum_cell=1, zeros_disclosive=False) - verdict, *_ = disclosure.check_quantile(30, 2, 20, rule, lambda *_: True) - assert verdict.status == "A" - verdict, *_ = disclosure.check_quantile(30, 0, 20, rule, lambda *_: True) - assert verdict.status == "F" # nothing below the witness is a zero rank, not a small one - - -@pytest.mark.parametrize( - ("status", "kind", "written"), - [ - ("A", "marginalHistogram", "C"), - ("A", "levelHistogram", "C"), - ("D", "levelHistogram", "C"), - ("C", "marginalHistogram", "C"), - ("F", "marginalHistogram", "F"), - ("F", "levelHistogram", "F"), - ("A", "mean", "A"), - ("D", "mean", "D"), - ("A", "support", "A"), - ("A", "quantile", "C"), - ("D", "quantile", "C"), - ("O", "mean", "O"), - ], -) -def test_a_count_table_or_a_quantile_writes_c_for_whatever_withheld_a_value(status, kind, written): - assert disclosure.written_status(status, kind) == written - - -def test_one_withheld_code_hides_the_size_a_total_gives_back(): - # PROFILE-SPEC section 3: cells of 10, 9, 9 and 10 records, written F, A, A, F under a - # known total, give both withheld cells as 9; written F, C, C, F they read as a code - # that names no check. - cells = disclosure.publish_cells([10, 9, 9, 10], ACRO_DEFAULT) - assert cells.statuses == ("F", "A", "A", "F") - written = [disclosure.written_status(code, "levelHistogram") for code in cells.statuses] - assert written == ["F", "C", "C", "F"] +def test_the_counts_a_check_for_edges_writes_are_the_ones_the_summary_names(): + # PROFILE-SPEC section 3: a histogram's or a quantile's withheld groups and cells are all + # written C, so the report says so for exactly these definitions. assert disclosure.ONE_CODE == {"marginalHistogram", "levelHistogram", "quantile"} diff --git a/tests/profiler/test_document.py b/tests/profiler/test_document.py index cc4e2552..a2864e1b 100644 --- a/tests/profiler/test_document.py +++ b/tests/profiler/test_document.py @@ -15,6 +15,7 @@ from fluid_build.profiler.disclosure import ACRO_DEFAULT from fluid_build.profiler.document import ( SCHEMA_URL, + SCHEMA_URL_1_1_0, DocumentError, build_document, canonical_bytes, @@ -79,6 +80,28 @@ def test_a_built_document_is_format_1_0_0(): assert document["disclosureRules"] == {"acro-default": ACRO_DEFAULT.to_document()} +def _with_statistic(definition): + table = _table() + table["columns"]["amount"]["statistics"] = [{"definition": definition, "groups": []}] + return table + + +def test_a_quantile_with_bins_needs_format_1_1_0_and_nothing_else_does(): + assert _document(tables={"orders": _with_statistic("quantile/2")})["_schemaURL"] == ( + SCHEMA_URL_1_1_0 + ) + for definition in ("mean/1", "marginalHistogram/1", "levelHistogram/1", "quantile/1"): + document = _document(tables={"orders": _with_statistic(definition)}) + assert document["_schemaURL"] == SCHEMA_URL + assert _document()["_schemaURL"] == SCHEMA_URL + + +def test_the_formats_are_the_ones_flux_stats_reads(): + profile = pytest.importorskip("flux_stats.profile") + assert SCHEMA_URL == profile.SCHEMA_URL + assert SCHEMA_URL_1_1_0 == profile.SCHEMA_URL_1_1_0 + + def test_the_producer_names_this_repository_at_the_release(): assert producer_uri("0.31.0") == ( "https://github.com/Agenticstiger/forge-cli/tree/v0.31.0/fluid_build/profiler" diff --git a/tests/profiler/test_flux_stats_backend.py b/tests/profiler/test_flux_stats_backend.py index 2f821994..030b11bc 100644 --- a/tests/profiler/test_flux_stats_backend.py +++ b/tests/profiler/test_flux_stats_backend.py @@ -28,6 +28,7 @@ import pytest from fluid_build.profiler._stats import ( + BundleTextRefused, DocumentRefused, ReadRefused, StatisticsBackend, @@ -285,3 +286,32 @@ def test_the_mean_a_profile_carries_is_the_one_the_check_compares(tmp_path, back assert entry.definition.compare(same, reference, check) higher = GroupStatistic(group="", k=reference.k, sum_x=reference.sum_x * 12 // 10) assert not entry.definition.compare(higher, reference, check) + + +def test_the_rounding_status_is_the_registrys_word_for_it(): + from flux_stats import edge_counts + + from fluid_build.profiler._stats import WITHHELD_ROUNDING + + assert WITHHELD_ROUNDING == edge_counts.ROUNDING + + +def test_the_backend_reads_a_bundle_through_the_registrys_exact_reader(): + backend = load_backend() + budget = backend.read_budget() + (document,) = backend.read_documents("rate: 0.50\nwhen: 2024-01-01\n", name="s", budget=budget) + assert document == {"rate": Decimal("0.50"), "when": "2024-01-01"} + assert backend.canonicalise(document, name="s") == {"rate": "0.50", "when": "2024-01-01"} + + +def test_two_keys_that_read_as_one_are_refused_without_naming_the_key(): + # A bundle key may hold anything, so the refusal of two source keys that read as one + # canonical key (FLX-1007) says the code's title, not the key. + backend = load_backend() + composed, decomposed = chr(0xE9), "e" + chr(0x301) # one key once normalised + with pytest.raises(BundleTextRefused) as refused: + backend.canonicalise( + {f"{composed}-hunter2": 1, f"{decomposed}-hunter2": 2}, name="simulation.flux.yml" + ) + said = str(refused.value) + assert "FLX-1007" in said and "simulation.flux.yml" in said and "hunter2" not in said diff --git a/tests/profiler/test_flux_stats_pin.py b/tests/profiler/test_flux_stats_pin.py index 73c6170e..1e5a492d 100644 --- a/tests/profiler/test_flux_stats_pin.py +++ b/tests/profiler/test_flux_stats_pin.py @@ -24,6 +24,10 @@ def _profile_flux_stats() -> list: return [r for r in requirements if g.requirement_name(r) == "flux-stats"] +def test_the_requirement_names_the_yaml_extra_for_the_bundle_reader(): + assert Requirement(_stats.FLUX_STATS_REQUIREMENT).extras == {"yaml"} + + def test_the_profile_extra_names_flux_stats_once(): assert len(_profile_flux_stats()) == 1 @@ -36,11 +40,11 @@ def test_the_requirement_is_a_range_of_released_versions(): requirement = Requirement(_stats.FLUX_STATS_REQUIREMENT) assert requirement.url is None # PyPI refuses a URL in Requires-Dist assert requirement.marker is None # flux-stats supports Python 3.10, the floor here - assert requirement.specifier.contains("0.6.2") - assert requirement.specifier.contains("0.6.9") + assert requirement.specifier.contains("0.7.0") + assert requirement.specifier.contains("0.7.9") assert not requirement.specifier.contains("0.0.1") # the placeholder holds no code - assert not requirement.specifier.contains("0.6.1") # no flux_stats.fidelity for --bundle - assert not requirement.specifier.contains("0.7.0") # a 0.x minor may rename what is imported + assert not requirement.specifier.contains("0.6.2") # no edge_counts, exact_yaml or quantile/2 + assert not requirement.specifier.contains("0.8.0") # a 0.x minor may rename what is imported def test_the_install_command_installs_the_extra_that_holds_flux_stats(): diff --git a/tests/profiler/test_runner.py b/tests/profiler/test_runner.py index c203f9f6..69d776f5 100644 --- a/tests/profiler/test_runner.py +++ b/tests/profiler/test_runner.py @@ -296,7 +296,7 @@ def version(name): with pytest.raises(StatisticsUnavailable) as caught: load_backend() message = str(caught.value) - assert f"and flux-stats {state}. It needs flux-stats>=0.6.2,<0.7;" in message + assert f"and flux-stats {state}. It needs flux-stats[yaml]>=0.7.0,<0.8;" in message assert "install it with: pip install 'data-product-forge[profile]'" in message