From 2fc759e16ccce6de8a4aebeb170a245c279db013 Mon Sep 17 00:00:00 2001 From: Speculator55005 <50082482+fas89@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:21:22 +0100 Subject: [PATCH] feat(profile): children per parent for every declared relationship When one call profiles both exposes of a semantics.relationships entry, without --bundle, the profile's existing relationships[] (PROFILE-SPEC section 5, format 1-0-0/1-1-0 unchanged) carries a marginalHistogram/1 with the parent as the unit: each parent row counts once, by its number of child rows, on the bins 0, 1, 2, 3-4, 5-9, 10-19 and 20 or more. The last bin ends at 2^53 - 1, the largest count a profile holds, so a parent with many children is counted where it belongs and no value is cut to fit. - keys are matched as the registry reads both columns, so 7 and "007" agree where the type is a number; a null key, on either side, matches none; - the histogram is formed by the registry's own definition (StatisticsBackend.count_histogram) and checked by check_edge_counts in a call of its own, as flux-stats 0.7.0 documents for a relationship's count of children, then spelled as a column's histogram is; - the child's foreign-key columns withhold their distinctCount, sum, min, max and quantiles (D) and form no mean/1: the parents with a child, beside the parents' row count, give those with none, and the key's own values describe each parent's children key by key; - a relationship whose target is not in the call, declares a validity window, or whose keys are read in different domains, or whose fromColumns and toColumns are not two lists of one length, is named and left out; - the key columns of a counted relationship take no default statistic of their own: a key's level histogram, histogram or quantiles count each key's rows, and the child key's rows are each parent's children; - relationships[] lists them in the contract's order, whatever order the call names the exposes in, so the document's bytes do not depend on it. The output check sees the parents' cells and their total only. A reader can know more: other tables over the same parents (an enum's level histogram, another relationship's cells) bound their number, and the children's own counts bound what they hold, each child row having at most one parent and each parent between its bin's edges. disclosure.pinned_children_cells takes the strongest reader, who knows the numbers of parents, of children and of parents whose key is null exactly, besides the published bands and the check's own rule (the withheld cells never total a small count). A withheld cell stands only where two tables that reader can hold true differ in it, found by moving one parent and, if the children's bounds break, as many more as repair them; the search errs toward withholding. runner._audited_children withholds the published cell of the highest edge until no cell is pinned, down to the whole histogram. The disclosure audit splits a band of children over a band of parents every way, children unbounded and none an orphan, under ACRO's rule: 875 and 4,700 worlds in the suite and up to 144,962 beside it pin nothing, where the check alone left 14 pins in the first. Keyed by the exact numbers of parents and children, up to 7 parents and 14 children pin nothing under every scaled rule, exact counts included, so the exact-count case is closed too. Tests cover an enum over the parents beside the relationship, two relationships into one parent, and enum and number keys. ProfileResult is read through its tables (and relationships) only: the one-table shortcuts kept beside them (plan, statistics, withheld_statistics, partly_withheld_statistics, left_out) go, and the tests read tables[0]. --- AGENTS.md | 2 +- CHANGELOG.md | 32 ++ fluid_build/cli/profile_cmd.py | 23 +- fluid_build/profiler/_stats.py | 31 ++ fluid_build/profiler/disclosure.py | 126 +++++- fluid_build/profiler/document.py | 14 +- fluid_build/profiler/runner.py | 388 ++++++++++++++++-- tests/profiler/_fake_backend.py | 6 +- tests/profiler/test_bundle_flux_stats.py | 30 +- tests/profiler/test_disclosure_oracle.py | 256 +++++++++++- tests/profiler/test_flux_stats_backend.py | 2 +- tests/profiler/test_relationships.py | 468 ++++++++++++++++++++++ tests/profiler/test_runner.py | 8 +- 13 files changed, 1319 insertions(+), 67 deletions(-) create mode 100644 tests/profiler/test_relationships.py diff --git a/AGENTS.md b/AGENTS.md index eae7965b..f8230c36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,7 +164,7 @@ forge-cli/ | Bundle builder | `fluid_build/forge/core/bundle.py` — deterministic tgz + MANIFEST + fragment extraction + `$source` sentinel | | Build runners | `fluid_build/build_runners/{dbt,python}/` — extracted from legacy `cli/execute.py` in Phase 1 | | Masking at landing | `fluid_build/build_runners/_masking.py` — the DuckDB runner applies `exposes[].policy.privacy.masking[]` inside the COPY (file, S3, BigQuery staging, DLQ); secrets only from env (`FLUID_PII_HASH_SECRET`, `FLUID_PII_TOKENIZATION_KEY`, `FLUID_PII_ENCRYPTION_SECRET_KEY`, or `params.saltEnv`/`keyEnv`), held in UDF closures, never in SQL; an unappliable rule refuses the build, and so does a DuckDB without numpy (its `create_function` needs it; `[local]` names numpy, and the refusal names the extra). `cli/_verify_masking.py` is stage 9's shape check (local DuckDB, Athena in the count query) | -| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads each expose's records where they live, every expose of one call into one document (`runner.profile_exposes`; the CLI pairs `--source =` and `--table =` with repeated `--expose`) (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC document, format 1-0-0 or, once it holds a `quantile/2`, 1-1-0 (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). Without `--bundle` it declares, from the contract alone, a fidelity block of the statistics `fluxsim init` reads (`defaults.py`: a `levelHistogram/1` over every level of an `enum(...)` column, a `marginalHistogram/1` over a `validationRules` range of two ends inside the type (each value checked against the range exactly, `DeclaredRange`), else `quantile/2` on a ladder fixed by the type for numbers and dates, grouped level histograms per categorical `semantics.dimensions` level, at most `defaults.MOST_GROUPED_CELLS` cells per column; `DeclaredBlock.every_declared_group` lists every declared level as a group) and answers it through the same path; `tests/profiler/test_disclosure_oracle.py` enumerates every tiny world to show no new statistic pins a protected count, which is why a column never gets a histogram and quantiles together and why `runner._counted_levels` / `_binned_numbers` withhold `distinctCount`, and a declared range's `sum`, `min`, `max` and `quantiles`. With `--bundle DIR` it answers the fidelity block of DIR's one Simulation instead: `bundle.py` reads `*.flux.yml` as the engine does (flux-stats' `flux_stats.exact_yaml` and `flux_stats.canonical`, through the seam, one `ReadBudget` per bundle), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a quantile with bins is `quantile/2` (the profile then names format 1-1-0), the histograms and quantiles of one column go to flux-stats' `check_edge_counts` in one call (FLUX ADR 0012; `runner._edge_decisions`), a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | +| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads each expose's records where they live, every expose of one call into one document (`runner.profile_exposes`; the CLI pairs `--source =` and `--table =
` with repeated `--expose`) (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC document, format 1-0-0 or, once it holds a `quantile/2`, 1-1-0 (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). Without `--bundle` it declares, from the contract alone, a fidelity block of the statistics `fluxsim init` reads (`defaults.py`: a `levelHistogram/1` over every level of an `enum(...)` column, a `marginalHistogram/1` over a `validationRules` range of two ends inside the type (each value checked against the range exactly, `DeclaredRange`), else `quantile/2` on a ladder fixed by the type for numbers and dates, grouped level histograms per categorical `semantics.dimensions` level, at most `defaults.MOST_GROUPED_CELLS` cells per column; `DeclaredBlock.every_declared_group` lists every declared level as a group) and answers it through the same path; `tests/profiler/test_disclosure_oracle.py` enumerates every tiny world to show no new statistic pins a protected count, which is why a column never gets a histogram and quantiles together and why `runner._counted_levels` / `_binned_numbers` withhold `distinctCount`, and a declared range's `sum`, `min`, `max` and `quantiles`. A `semantics.relationships` entry between two exposes of one call gets children per parent in the document's `relationships[]` (`runner._children_per_parent`, bins `runner.CHILDREN_EDGES`, the parent as unit, its own `check_edge_counts` call), and the child's foreign key withholds its `distinctCount`, `sum`, `min`, `max` and `quantiles` and forms no `mean/1`; `runner._audited_children` then withholds every cell the strongest reader could work out: the numbers of parents, children and null-key parents known exactly, the check's own rule included (`disclosure.pinned_children_cells`, a witness search that errs toward withholding); the key columns of a counted relationship take no default statistic (`runner._relationship_keys`), since a key's own histogram or quantiles would count each parent's children. With `--bundle DIR` it answers the fidelity block of DIR's one Simulation instead: `bundle.py` reads `*.flux.yml` as the engine does (flux-stats' `flux_stats.exact_yaml` and `flux_stats.canonical`, through the seam, one `ReadBudget` per bundle), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a quantile with bins is `quantile/2` (the profile then names format 1-1-0), the histograms and quantiles of one column go to flux-stats' `check_edge_counts` in one call (FLUX ADR 0012; `runner._edge_decisions`), a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | | CI template generator | `fluid_build/forge/core/pipeline_templates.py` — 7-system generator; Jenkins ships the full 11-stage parameterized template with `--install-mode {pypi,dev-source}` | | Forge (AI creation) | `fluid_build/forge/` (templates, generators, extensions) | | Log redaction (global) | `fluid_build/observability/secret_redactor.py` — `SecretRedactingFilter` wired into Python logging | diff --git a/CHANGELOG.md b/CHANGELOG.md index f178da46..70eb66ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`fluid profile` counts children per parent for each declared relationship.** When one call + profiles both exposes of a `semantics.relationships` entry, without `--bundle`, the + document's `relationships[]` (PROFILE-SPEC §5, format unchanged) carries a + `marginalHistogram/1` with the parent as unit: each parent row counts once, by its number of + child rows, on the bins 0, 1, 2, 3 to 4, 5 to 9, 10 to 19, and 20 or more. The last bin ends + at 2^53 - 1, the largest count a profile holds, so no count is cut to fit. Keys are matched + as the registry reads them, and a null key matches none. The histogram goes through + `check_edge_counts` in a call of its own, as flux-stats documents for a relationship's count + of children. The child's foreign-key columns withhold their `distinctCount`, `sum`, `min`, + `max` and `quantiles` under `D` and form no `mean/1`: the parents that have a child, beside + the parents' row count, would give those with none, and the key's own values describe each + parent's children key by key. A relationship whose target is not profiled in the call, + declares a validity window, or whose keys are read in different domains is named in the + summary and left out. The key columns + of a counted relationship take no default statistic of their own: a key's level histogram, + histogram or quantiles count each key's rows, and the child key's rows are each parent's + children. The check sees the parents' cells and their total, while a reader can know more: + other tables over the same parents (an `enum` column's level histogram, another + relationship's cells) bound their number, and the children's own counts bound what they + hold, each child row having at most one parent and each parent between its bin's edges. + So each withheld cell is audited against the strongest reader, who knows the number of + parents, of children and of parents whose key is null exactly, besides the published + bands and the check's own rule: a withheld cell stands only where two tables that reader + can hold true differ in it. While one cannot be shown to, the published cell of the + highest edge is withheld too, down to the whole histogram, which is where parents with + few children often land. Audited world by world with children unbounded and none an + orphan: under ACRO's rule no split of a band of children over a band of parents pins a + count (up to 144,962 worlds), and up to 7 parents keyed by their exact numbers pin none + under every scaled rule, exact counts included. + ## [0.36.0] — 2026-11-19 ### Added diff --git a/fluid_build/cli/profile_cmd.py b/fluid_build/cli/profile_cmd.py index fe93f97e..35996448 100644 --- a/fluid_build/cli/profile_cmd.py +++ b/fluid_build/cli/profile_cmd.py @@ -68,7 +68,14 @@ date or a timestamp); - for each categorical semantics.dimensions entry on an enum column, the other enum columns' histograms per level of it, every declared level a - group, up to 10,000 cells for one column's groupings. + group, up to 10,000 cells for one column's groupings; + - for each semantics.relationships entry between two exposes of the call, + children per parent: how many child rows each parent row has, counted on + the bins 0, 1, 2, 3-4, 5-9, 10-19 and 20 or more, with the parent as the + unit; the child's foreign key then withholds its distinctCount, sum, + min, max and quantiles, its key columns take no statistic of their own, + and a cell a reader who knows the numbers of parents and children + exactly could work out is withheld too. A record outside what the contract declares (a level its type does not list, a value outside its type or its declared range) refuses the profile. @@ -734,6 +741,20 @@ def _report(say: Any, result: Any, out: Path, *, answered_block: bool = False) - ) for table in tables: _report_table(say, table, f"exposes[{table.plan.expose_id}]: " if several else "", grid) + for counted in result.relationships: + link = counted.relationship + shown = ( + f"{counted.published} of {counted.cells} cells published" + if counted.published + else "withheld whole" + ) + say( + f"Children per parent of exposes[{link.parent}] in exposes[{link.child}] " + f"({link.name}): {shown}.", + soft_wrap=True, + ) + for child, name, reason in result.relationships_left_out: + say(f"No children per parent for exposes[{child}] {name!r}: {reason}.", soft_wrap=True) if not result.published: say("Every value was withheld: the document says nothing about the records.") say(f"Wrote {out} (readable by you alone; read it before you share it).", soft_wrap=True) diff --git a/fluid_build/profiler/_stats.py b/fluid_build/profiler/_stats.py index 6c60137e..21a33c59 100644 --- a/fluid_build/profiler/_stats.py +++ b/fluid_build/profiler/_stats.py @@ -396,6 +396,15 @@ def measure( record the block's declaration refuses. """ + def count_histogram( + self, counts: Sequence[int], edges: Sequence[int], fractional_bits: int + ) -> MeasuredGroup: + """``marginalHistogram/1`` over whole numbers, one per unit: the ungrouped group. + + *counts* are one value per unit (a parent's number of children), and *edges* the + bins' whole-number edges, placed on the grid as the registry places any edge. + """ + def check_edge_counts( self, statistics: Sequence[EdgeStatistic], @@ -522,6 +531,7 @@ def __init__(self) -> None: from flux_stats.reading import ValueDomain, category, read_exact self._metrics = metrics + self._parameters = Parameters self._statistics = tuple(_Statistic(each, Parameters()) for each in definitions()) self._grid = Grid self._refused = FluxError @@ -823,6 +833,27 @@ def _measured( ) return replace(group, values=tuple(values), encoded=tuple(encoded)) + def count_histogram( + self, counts: Sequence[int], edges: Sequence[int], fractional_bits: int + ) -> MeasuredGroup: + grammar = self._fidelity() + grid = self._grid(fractional_bits) + histogram = grammar.definition_named("marginalHistogram/1") + encoded = tuple(grid.quantise(Fraction(edge), subject="bin edge") for edge in edges) + state = histogram.measure( + "", + [grid.quantise(Fraction(count), subject="count") for count in counts], + self._parameters(edges=encoded), + ) + spelled: Dict[str, Any] = state.to_document(histogram) + return MeasuredGroup( + key="", + document=spelled, + k=state.k, + statistic=state, + cells=tuple(state.cell_counts("marginalHistogram")), + ) + def check_edge_counts( self, statistics: Sequence[EdgeStatistic], diff --git a/fluid_build/profiler/disclosure.py b/fluid_build/profiler/disclosure.py index 44106fe7..2d3267eb 100644 --- a/fluid_build/profiler/disclosure.py +++ b/fluid_build/profiler/disclosure.py @@ -113,7 +113,19 @@ import unicodedata from dataclasses import dataclass from fractions import Fraction -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union +from typing import ( + Any, + Callable, + Dict, + FrozenSet, + Iterable, + List, + Optional, + Sequence, + Set, + Tuple, + Union, +) __all__ = [ "ACRO_DEFAULT", @@ -138,6 +150,7 @@ "check_rounding", "check_sum", "check_support", + "pinned_children_cells", "round_count", "sample_size", "suppress_secondary", @@ -533,3 +546,114 @@ def suppress_secondary(verdicts: Sequence[Verdict]) -> Tuple[Verdict, ...]: return tuple(verdicts) withheld = Verdict(SECONDARY, "another group of the statistic is withheld") return tuple(withheld if verdict.published else verdict for verdict in verdicts) + + +def pinned_children_cells( + lower_edges: Sequence[int], + cells: Sequence[int], + published: Sequence[bool], + children: int, + rule: DisclosureRule, + childless: int = 0, +) -> FrozenSet[int]: + """The withheld cells of a children-per-parent table a reader may work out exactly. + + The table counts parents by their number of children, on bins whose lower edges are + *lower_edges* (the first 0; each bin ends below the next, the last has no top). The + reader is the strongest a profile can meet: it knows the exact number of parents, + the exact number of children they hold together (*children*), how many parents hold + none because their key is null (*childless*), each published cell's band, and the + output check's own rule, so that where a cell is published the withheld cells + together are not a small count, nor a zero where zeros are disclosive. Any table it + can hold true has nonnegative cells with the parents' sum, ``lower * c <= children + <= upper * c`` (a parent holds between its bin's edges), at least *childless* parents + in the first cell, and each published cell in its band. + + A withheld cell is safe when two such tables differ in it. The true table (*cells*) + is one; the other is searched for among the tables one or two parents' moves away. + A cell with no such pair is returned, so the search errs toward withholding: a + returned cell may be one no reader could pin after all. + """ + count = len(cells) + tops: List[Optional[int]] = [edge - 1 for edge in lower_edges[1:]] + [None] + bands = [ + sample_size(value, rule) if shown else None + for value, shown in zip(cells, published, strict=True) + ] + any_published = any(published) + + def holds(table: Sequence[int]) -> bool: + if min(table) < 0 or table[0] < childless: + return False + for value, band in zip(table, bands, strict=True): + if band is not None and not band["min"] <= value <= band["max"]: + return False + if any_published: + rest = sum(value for value, shown in zip(table, published, strict=True) if not shown) + if not check_count(rest, rule).published: + return False + if sum(edge * value for edge, value in zip(lower_edges, table, strict=True)) > children: + return False + if tops[-1] is None and table[-1] > 0: + return True + return sum((top or 0) * value for top, value in zip(tops, table, strict=True)) >= children + + withheld = [index for index, shown in enumerate(published) if not shown] + if not holds(cells): + return frozenset(withheld) # the reader's model misses these records: no claim + moves = [ + (source, target) for source in range(count) for target in range(count) if source != target + ] + + def repairs(table: Sequence[int], source: int, target: int) -> Set[int]: + """One parent, or as many moved *source* to *target* as the children's bounds need.""" + many = {1} + low = sum(edge * value for edge, value in zip(lower_edges, table, strict=True)) + if low > children and lower_edges[source] > lower_edges[target]: + gap = lower_edges[source] - lower_edges[target] + many.add(-(-(low - children) // gap)) + unbounded = tops[-1] is None and table[-1] > 0 + if not unbounded: + high = sum((top or 0) * value for top, value in zip(tops, table, strict=True)) + if high < children and tops[target] is None: + many.add(1) + elif high < children and (tops[target] or 0) > (tops[source] or 0): + gap = (tops[target] or 0) - (tops[source] or 0) + many.add(-(-(children - high) // gap)) + return {each for each in many if 1 <= each <= table[source]} + + pinned: Set[int] = set() + for index in withheld: + if not _two_tables(cells, index, moves, holds, repairs): + pinned.add(index) + return frozenset(pinned) + + +def _two_tables( + cells: Sequence[int], + index: int, + moves: Sequence[Tuple[int, int]], + holds: Callable[[Sequence[int]], bool], + repairs: Callable[[Sequence[int], int, int], Set[int]], +) -> bool: + """Whether a table two moves from *cells* holds and differs at *index*. + + The first move takes one parent into or out of *index*; the second moves one parent, + or as many as the children's bounds then need (*repairs*), between any two cells. + """ + for source, target in moves: + if index not in (source, target): + continue + first = list(cells) + first[source] -= 1 + first[target] += 1 + if first[index] != cells[index] and holds(first): + return True + for again_source, again_target in moves: + for many in repairs(first, again_source, again_target): + second = list(first) + second[again_source] -= many + second[again_target] += many + if second[index] != cells[index] and holds(second): + return True + return False diff --git a/fluid_build/profiler/document.py b/fluid_build/profiler/document.py index f4c309a3..65aa5093 100644 --- a/fluid_build/profiler/document.py +++ b/fluid_build/profiler/document.py @@ -27,7 +27,7 @@ import hashlib import unicodedata -from typing import Any, Dict, List, Mapping +from typing import Any, Dict, List, Mapping, Sequence from fluid_build.profiler.disclosure import MAX_COUNT, DisclosureRule @@ -103,18 +103,26 @@ def build_document( rule: DisclosureRule, tables: Mapping[str, Mapping[str, Any]], fractional_bits: int = DEFAULT_FRACTIONAL_BITS, + relationships: Sequence[Mapping[str, Any]] = (), ) -> Dict[str, Any]: - """Assemble a document from tables already checked by *rule*, in the format they need.""" + """Assemble a document from tables already checked by *rule*, in the format they need. + + *relationships* are PROFILE-SPEC §5's children per parent, also checked by *rule*; + a document without any has no ``relationships`` key, as before there were any. + """ check_fractional_bits(fractional_bits) if not tables: raise DocumentError("tables holds at least one table") - return { + built: Dict[str, Any] = { "_producer": producer, "_schemaURL": schema_url_for(tables), "encoding": {"fractionalBits": fractional_bits, "rounding": "floor"}, "disclosureRules": {rule.name: rule.to_document()}, "tables": {expose_id: dict(table) for expose_id, table in tables.items()}, } + if relationships: + built["relationships"] = [dict(each) for each in relationships] + return built def _check(value: Any, where: List[str]) -> None: diff --git a/fluid_build/profiler/runner.py b/fluid_build/profiler/runner.py index 09609398..3e0ee1bf 100644 --- a/fluid_build/profiler/runner.py +++ b/fluid_build/profiler/runner.py @@ -39,7 +39,7 @@ from datetime import date from fractions import Fraction from pathlib import Path -from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Sequence, Set, Tuple from fluid_build.policy._common import iter_exposes from fluid_build.profiler import defaults, disclosure, document, sources @@ -64,9 +64,12 @@ "DEFAULT_QUANTILE_LEVELS", "ColumnPlan", "ExposePlan", + "CHILDREN_EDGES", "ExposeSource", "ProfileError", "ProfileResult", + "Relationship", + "RelationshipResult", "TableResult", "column_domain", "expose_ids", @@ -204,6 +207,33 @@ class TableResult: uncovered: Tuple[Tuple[str, str], ...] = () +#: The bins a parent's number of children is counted in: 0, 1, 2, 3 to 4, 5 to 9, 10 to 19, +#: and 20 or more. The last bin ends at the largest count a profile holds, 2^53 - 1, so it +#: holds every parent with 20 children or more and no count is cut to fit. +CHILDREN_EDGES: Tuple[int, ...] = (0, 1, 2, 3, 5, 10, 20, disclosure.MAX_COUNT) + + +@dataclass(frozen=True) +class Relationship: + """A declared relationship whose two exposes one profile reads: a child's foreign key.""" + + child: str + name: str + parent: str + child_columns: Tuple[str, ...] + parent_columns: Tuple[str, ...] + + +@dataclass(frozen=True) +class RelationshipResult: + """What the profile says about one relationship's children per parent, for the summary.""" + + relationship: Relationship + #: The cells published, of the histogram's cells; 0 when it is withheld whole. + published: int + cells: int + + @dataclass(frozen=True) class ProfileResult: document: Dict[str, Any] @@ -221,33 +251,10 @@ class ProfileResult: withheld_cells: Mapping[str, int] = field(default_factory=dict) #: The grid the document declares and its on-grid statistics were formed on. fractional_bits: int = DEFAULT_FRACTIONAL_BITS - - def _only(self) -> TableResult: - if len(self.tables) != 1: - raise ValueError("a profile of several exposes has one result per expose: read .tables") - return self.tables[0] - - # The one expose of a profile of one expose, as these were before several could be read. - - @property - def plan(self) -> ExposePlan: - return self._only().plan - - @property - def statistics(self) -> Tuple[NamedStatistic, ...]: - return self._only().statistics - - @property - def withheld_statistics(self) -> Tuple[NamedStatistic, ...]: - return self._only().withheld_statistics - - @property - def partly_withheld_statistics(self) -> Tuple[PartlyNamedStatistic, ...]: - return self._only().partly_withheld_statistics - - @property - def left_out(self) -> Tuple[Tuple[str, str], ...]: - return self._only().left_out + #: Each relationship counted, children per parent. + relationships: Tuple[RelationshipResult, ...] = () + #: Each declared relationship not counted: the child, the relationship, and why. + relationships_left_out: Tuple[Tuple[str, str, str], ...] = () def _base_type(type_name: Any) -> str: @@ -587,11 +594,20 @@ def profile_exposes( grids[plan.expose_id] = grid fractional_bits = _one_grid(grids, blocks, fractional_bits) uncovered: List[Tuple[Tuple[str, str], ...]] = [() for _ in plans] + relationships, relationships_left_out = _declared_relationships(contract, plans, blocks) + held = _relationship_keys(relationships) for position, plan in enumerate(plans): if plan.expose_id not in blocks: declared[position], uncovered[position] = _default_block( - contract, plan, backend, fractional_bits + contract, plan, backend, fractional_bits, held.get(plan.expose_id, frozenset()) ) + edges = { + plan.expose_id: _edge_counted(found) for plan, found in zip(plans, declared, strict=True) + } + relationships, relationships_left_out = _keys_not_counted_at_edges( + relationships, relationships_left_out, edges + ) + keys = _key_columns(relationships) reads = [ _dates_read( plan, @@ -613,9 +629,15 @@ def profile_exposes( rule, tallies, fractional_bits, + keys.get(plan.expose_id, frozenset()), ) tables_doc[plan.expose_id] = table_doc results.append(replace(result, uncovered=missing)) + by_expose = {plan.expose_id: (plan, read) for plan, read in zip(plans, reads, strict=True)} + counted = [ + _children_per_parent(each, by_expose, backend, rule, tallies, fractional_bits) + for each in relationships + ] if producer is None: from fluid_build import __version__ @@ -625,6 +647,7 @@ def profile_exposes( rule=rule, tables=tables_doc, fractional_bits=fractional_bits, + relationships=[entry for entry, _ in counted], ) canonical = document.canonical_bytes(built) backend.check_document(canonical.decode("utf-8")) @@ -639,6 +662,8 @@ def profile_exposes( withheld=dict(sorted(tally.items())), withheld_cells=dict(sorted(cell_tally.items())), fractional_bits=fractional_bits, + relationships=tuple(result for _, result in counted), + relationships_left_out=tuple(relationships_left_out), ) @@ -676,20 +701,303 @@ def _as_day(value: Any) -> Any: return value +def _declared_relationships( + contract: Mapping[str, Any], + plans: Sequence[ExposePlan], + blocks: Mapping[str, BundleBlock], +) -> Tuple[List[Relationship], List[Tuple[str, str, str]]]: + """The ``semantics.relationships`` this profile counts children per parent for. + + A relationship is counted when the profile reads both its exposes, without a bundle + (a bundle's profile answers its block alone), and both keys are columns it reads in + one domain each, pair by pair. One whose target declares a validity window is not: + a parent there has a row per version. Every other is returned with why, never + guessed at. Both come in the contract's order, whatever order the call names the + exposes in, so the document's bytes do not depend on it. + """ + from fluid_build.data_rules._model import declares_validity_window + + exposes = { + str(item.get("exposeId")): item + for item in iter_exposes(dict(contract)) + if item.get("exposeId") + } + planned = {plan.expose_id: plan for plan in plans} + counted: List[Relationship] = [] + left_out: List[Tuple[str, str, str]] = [] + for plan in (planned[expose_id] for expose_id in exposes if expose_id in planned): + semantics = exposes[plan.expose_id].get("semantics") + declared = semantics.get("relationships") if isinstance(semantics, Mapping) else None + for raw in declared if isinstance(declared, list) else (): + if not isinstance(raw, Mapping): + continue + name, target = str(raw.get("name") or "?"), raw.get("to") + child_columns = _column_names(raw.get("fromColumns")) + parent_columns = _column_names(raw.get("toColumns")) + reason: Optional[str] = None + if not isinstance(target, str) or target not in planned: + reason = "its target is not profiled in this call: profile both exposes together" + elif plan.expose_id in blocks or target in blocks: + reason = "a profile that answers a bundle answers its fidelity block alone" + elif declares_validity_window(exposes[target]): + reason = "its target declares a validity window, so a parent has a row per version" + elif not child_columns or len(child_columns) != len(parent_columns): + reason = "its fromColumns and toColumns are not two lists of one length" + else: + reason = _keys_unread(plan, planned[target], child_columns, parent_columns) + if reason is not None: + left_out.append((plan.expose_id, name, reason)) + else: + counted.append( + Relationship(plan.expose_id, name, str(target), child_columns, parent_columns) + ) + return counted, left_out + + +def _column_names(value: Any) -> Tuple[str, ...]: + """The column names a relationship lists; none unless *value* is a list.""" + return tuple(str(each) for each in value) if isinstance(value, list) else () + + +def _keys_unread( + child: ExposePlan, + parent: ExposePlan, + child_columns: Sequence[str], + parent_columns: Sequence[str], +) -> Optional[str]: + """Why two keys cannot be matched as read, or ``None`` when every pair can be.""" + child_domains = {column.name: column.domain for column in child.columns} + parent_domains = {column.name: column.domain for column in parent.columns} + for mine, theirs in zip(child_columns, parent_columns, strict=True): + if mine not in child_domains or theirs not in parent_domains: + return f"key column {mine!r} or {theirs!r} is not one fluid profile reads" + if child_domains[mine] != parent_domains[theirs]: + return f"key columns {mine!r} and {theirs!r} are read in different domains" + return None + + +def _key_columns(relationships: Sequence[Relationship]) -> Dict[str, FrozenSet[str]]: + """The foreign-key columns of each child: their ``distinctCount`` is withheld. + + A foreign key's distinct values are the parents that have a child, so beside the + parent's row count it would give the parents with none, a cell of the histogram. + """ + found: Dict[str, Set[str]] = {} + for each in relationships: + found.setdefault(each.child, set()).update(each.child_columns) + return {child: frozenset(columns) for child, columns in found.items()} + + +def _relationship_keys(relationships: Sequence[Relationship]) -> Dict[str, FrozenSet[str]]: + """The key columns of every counted relationship, both sides, by expose. + + They take no default statistic: a key's own histogram or quantile counts each key's + rows, and the child key's rows are each parent's children, so beside children per + parent it would give a parent's count back. Their children per parent describe them. + """ + found: Dict[str, Set[str]] = {} + for each in relationships: + found.setdefault(each.child, set()).update(each.child_columns) + found.setdefault(each.parent, set()).update(each.parent_columns) + return {expose: frozenset(columns) for expose, columns in found.items()} + + +def _edge_counted(declared: Optional[DeclaredBlock]) -> FrozenSet[str]: + """The columns a statistic counts records at the edges of, or groups by.""" + if declared is None: + return frozenset() + found = set() + for statistic in declared.statistics: + if statistic.kind in _EDGE_KINDS: + found.add(statistic.column) + if statistic.group_by is not None: + found.add(statistic.group_by) + return frozenset(found) + + +def _keys_not_counted_at_edges( + relationships: Sequence[Relationship], + left_out: Sequence[Tuple[str, str, str]], + edges: Mapping[str, FrozenSet[str]], +) -> Tuple[List[Relationship], List[Tuple[str, str, str]]]: + """*relationships* less any whose key a statistic still counts at its edges. + + The default blocks leave every key out, so none should be; one that is would give + each parent's children beside the histogram, and is named and left out instead. + """ + kept: List[Relationship] = [] + out = list(left_out) + for each in relationships: + if set(each.child_columns) & edges.get(each.child, frozenset()) or set( + each.parent_columns + ) & edges.get(each.parent, frozenset()): + out.append((each.child, each.name, "a key column is counted at its edges")) + else: + kept.append(each) + return kept, out + + +def _children_per_parent( + relationship: Relationship, + tables: Mapping[str, Tuple[ExposePlan, sources.SourceColumns]], + backend: StatisticsBackend, + rule: disclosure.DisclosureRule, + tallies: Tuple["Counter[str]", "Counter[str]"], + fractional_bits: int, +) -> Tuple[Dict[str, Any], RelationshipResult]: + """The ``relationships[]`` entry of one relationship: a histogram with the parent as unit. + + Each parent row counts once, by the child rows whose key equals its own, compared as + the registry reads both (a parent whose key is null, like a child's null key, matches + none). The counts go through the output check for counts at edges in a call of their + own, as ``check_edge_counts`` documents for a relationship's count of children, and + are spelled as a column's histogram is. + """ + parents = _keys(tables[relationship.parent], relationship.parent_columns, backend) + children = Counter( + key + for key in _keys(tables[relationship.child], relationship.child_columns, backend) + if None not in key + ) + counts = [0 if None in key else children.get(key, 0) for key in parents] + encode = backend.encoder(fractional_bits) + statistic = DeclaredStatistic( + identifier="marginalHistogram/1", + kind="marginalHistogram", + column=f"{relationship.child}.{relationship.name}", + group_by=None, + shape={"bins": [str(edge) for edge in CHILDREN_EDGES]}, + columns=(), + read_as={}, + edges=tuple(encode(Fraction(edge)) for edge in CHILDREN_EDGES), + ) + measured = MeasuredStatistic( + groups=(backend.count_histogram(counts, CHILDREN_EDGES, fractional_bits),) + ) + try: + (decision,) = backend.check_edge_counts( + [_edge_statistic(statistic, measured, rule)], + total=_column_total(statistic, measured), + minimum_cell=rule.minimum_cell, + zeros_disclosive=rule.zeros_disclosive, + round_count=lambda count: disclosure.round_count(count, rule), + ) + except ValueError as exc: + raise ProfileError( + "profile_edge_counts_refused", + f"the children per parent of {statistic.column} cannot describe one set of " + f"parents: {exc}", + {"written": "nothing", "next": "report this: the output check refused its input"}, + ) from None + childless = sum(1 for key in parents if None in key) + decision = _audited_children(decision, measured.groups[0], sum(counts), childless, rule) + entry = _declared_entry(statistic, measured, rule, tallies, fractional_bits, decision) + (group,) = entry["groups"] + published = ( + sum(1 for count in group["binCounts"] if count is not None) + if group["status"] == disclosure.FREE + else 0 + ) + document_entry = { + "parent": {"table": relationship.parent, "columns": list(relationship.parent_columns)}, + "child": {"table": relationship.child, "columns": list(relationship.child_columns)}, + "statistics": [entry], + } + return document_entry, RelationshipResult( + relationship, published=published, cells=len(CHILDREN_EDGES) - 1 + ) + + +#: Why the audit of children per parent withholds a cell the output check let through. +_CHILDREN_PINNED = "the parents' and the children's counts would give a withheld cell back" + + +def _audited_children( + decision: EdgeDecision, + group: MeasuredGroup, + children: int, + childless: int, + rule: disclosure.DisclosureRule, +) -> EdgeDecision: + """*decision*, with more cells withheld until no withheld cell can be worked out. + + ``check_edge_counts`` sees the parents' cells and their total. A reader can know more: + the other tables over the same parents (an enum's level histogram, another + relationship's cells) bound their number, and the children's own counts bound the + children they hold, each child row having at most one parent. So the audit takes the + strongest reader, who knows the parents' number, their *children* and the *childless* + ones whose key is null exactly (:func:`disclosure.pinned_children_cells`). While it + can pin a withheld cell, the published cell of the highest lower edge is withheld + too, down to every cell, which withholds the histogram whole. It reads counts the + profiler formed, never a record's value, and writes only which cells are withheld. + """ + if decision.group_status[group.key] != disclosure.FREE: + return decision + lower = CHILDREN_EDGES[:-1] + statuses = list(decision.cell_status[group.key]) + reasons = list(decision.cell_reason[group.key]) + while any(status == disclosure.FREE for status in statuses): + published = [status == disclosure.FREE for status in statuses] + if not disclosure.pinned_children_cells( + lower, group.cells, published, children, rule, childless + ): + break + heaviest = max( + (index for index, shown in enumerate(published) if shown), + key=lambda index: lower[index], + ) + statuses[heaviest] = disclosure.WITHHELD + reasons[heaviest] = _CHILDREN_PINNED + return replace( + decision, + cell_status={**decision.cell_status, group.key: tuple(statuses)}, + cell_reason={**decision.cell_reason, group.key: tuple(reasons)}, + ) + + +def _keys( + table: Tuple[ExposePlan, sources.SourceColumns], + names: Sequence[str], + backend: StatisticsBackend, +) -> List[Tuple[Any, ...]]: + """Each row's key over *names*, every part read as the registry reads its column.""" + plan, read = table + domains = {column.name: column.domain for column in plan.columns} + columns = [ + [ + backend.read_value(domains[name], value, column=name, row_index=index) + for index, value in enumerate(read.values[name]) + ] + for name in names + ] + return list(zip(*columns, strict=True)) if columns else [] + + def _default_block( - contract: Mapping[str, Any], plan: ExposePlan, backend: StatisticsBackend, fractional_bits: int + contract: Mapping[str, Any], + plan: ExposePlan, + backend: StatisticsBackend, + fractional_bits: int, + keys: FrozenSet[str] = frozenset(), ) -> Tuple[Optional[DeclaredBlock], Tuple[Tuple[str, str], ...]]: """The block the profiler declares for *plan* from the contract alone, read by the registry. ``(None, ...)`` when the contract declares nothing to answer. The block is the - profiler's own, so a grammar refusal of it is a defect, and says so. + profiler's own, so a grammar refusal of it is a defect, and says so. The *keys* of + counted relationships take no statistic in it (:func:`_relationship_keys`). """ expose = next( item for item in iter_exposes(dict(contract)) if item.get("exposeId") == plan.expose_id ) - planned = defaults.default_plan(expose, plan.columns, backend.enum_levels, fractional_bits) + columns = [column for column in plan.columns if column.name not in keys] + held = tuple( + (column.name, "a key of a counted relationship, which its children per parent describe") + for column in plan.columns + if column.name in keys + ) + planned = defaults.default_plan(expose, columns, backend.enum_levels, fractional_bits) if planned.block is None: - return None, planned.uncovered + return None, planned.uncovered + held try: declared = backend.read_block(planned.block, contract, plan.expose_id) except BlockRefused as exc: @@ -704,7 +1012,7 @@ def _default_block( ) from None _check_declared_columns(declared, plan) block = replace(declared, every_declared_group=True, declared_ranges=planned.ranges) - return block, planned.uncovered + return block, planned.uncovered + held def _refuse_repeated(exposes: Sequence[ExposeSource]) -> None: @@ -765,12 +1073,18 @@ def _profile_table( rule: disclosure.DisclosureRule, tallies: Tuple["Counter[str]", "Counter[str]"], fractional_bits: int, + keys: FrozenSet[str] = frozenset(), ) -> Tuple[Dict[str, Any], TableResult]: - """One table of the document, from one read of its records, and its summary.""" + """One table of the document, from one read of its records, and its summary. + + *keys* are the table's foreign-key columns whose children per parent are counted. + """ tally, cell_tally = tallies - binned = _binned_numbers(declared, plan) + # A foreign key's own values are each parent's children, key by key: its sum, extremes + # and quantiles describe them as a binned number's do its bins, so it withholds them too. + binned = _binned_numbers(declared, plan) | keys replaced = _ungrouped_means(declared) | binned - counted = _counted_levels(declared) + counted = _counted_levels(declared) | keys columns_doc = { column.name: _column( column, diff --git a/tests/profiler/_fake_backend.py b/tests/profiler/_fake_backend.py index 2b53997f..07e1d1a6 100644 --- a/tests/profiler/_fake_backend.py +++ b/tests/profiler/_fake_backend.py @@ -19,7 +19,8 @@ The fidelity grammar has no stand-in: it is flux-stats' own (ADR 0009), and the profile every run answers without ``--bundle`` is a fidelity block the profiler declares from the contract (``fluid_build/profiler/defaults.py``). So ``read_block``, -``measure``, ``check_edge_counts`` and ``enum_levels`` hand the work to flux-stats. +``measure``, ``check_edge_counts``, ``count_histogram`` and ``enum_levels`` hand the work +to flux-stats. """ from __future__ import annotations @@ -207,6 +208,9 @@ def measure(self, block: Any, statistic: Any, rows: Any) -> Any: def check_edge_counts(self, statistics: Any, **checks: Any) -> Any: return self._real().check_edge_counts(statistics, **checks) + def count_histogram(self, counts: Any, edges: Any, fractional_bits: int) -> Any: + return self._real().count_histogram(counts, edges, fractional_bits) + def metrics(self) -> Tuple[FakeMetric, ...]: return METRICS diff --git a/tests/profiler/test_bundle_flux_stats.py b/tests/profiler/test_bundle_flux_stats.py index 2c2c2ffd..06fe145b 100644 --- a/tests/profiler/test_bundle_flux_stats.py +++ b/tests/profiler/test_bundle_flux_stats.py @@ -174,10 +174,10 @@ def _entry(result, column: str, definition: str, group_by: Optional[str] = None) def test_the_document_answers_every_declared_property(tmp_path): result = _profile(tmp_path, _rows()) assert result.fractional_bits == 16 - assert [reason for _, reason in result.left_out] == [ + assert [reason for _, reason in result.tables[0].left_out] == [ "a correlation's moments need the exact record count, which the output checks round" ] - assert result.left_out[0][0] == "correlation of 'amount' and 'qty'" + assert result.tables[0].left_out[0][0] == "correlation of 'amount' and 'qty'" assert not [ g for e in _entries(result, "amount") @@ -290,11 +290,11 @@ def test_a_rare_support_member_withholds_the_group_and_siblings_of_a_grouped_mea assert _entry(result, "amount", "mean/1")["groups"][0]["status"] == "F" # Three siblings of the mean, and the quantile beside the histogram of its column. assert result.withheld["D"] == 4 + 1 # + the groupBy column's distinctCount - assert ("amount", "mean/1", "market", None) not in result.statistics - assert ("amount", "mean/1", None, None) in result.statistics - assert ("market", "support/1", None, None) in result.withheld_statistics - assert ("amount", "mean/1", "market", None) in result.withheld_statistics - assert ("amount", "mean/1", None, None) not in result.withheld_statistics + assert ("amount", "mean/1", "market", None) not in result.tables[0].statistics + assert ("amount", "mean/1", None, None) in result.tables[0].statistics + assert ("market", "support/1", None, None) in result.tables[0].withheld_statistics + assert ("amount", "mean/1", "market", None) in result.tables[0].withheld_statistics + assert ("amount", "mean/1", None, None) not in result.tables[0].withheld_statistics # The histogram's DE cell is a one: withheld by its cell, the group is published. (group,) = _entry(result, "market", "levelHistogram/1")["groups"] assert group["cellStatus"][0] == "C" and group["levelCounts"][0] is None @@ -533,7 +533,7 @@ def test_a_quantile_with_bins_is_written_as_quantile_2(tmp_path): (group,) = read.groups assert group.published and group.statistic.quantile_bin == 0 assert (group.statistic.bin_below, group.statistic.bin_through) == (0, 70) - assert result.statistics.count( + assert result.tables[0].statistics.count( ("amount", "quantile/2", None, "at level 0.5 over bins 0, 50, 100") ) @@ -715,7 +715,7 @@ def test_a_null_key_on_a_null_value_hides_nothing(tmp_path): def test_two_quantiles_of_one_column_are_named_apart_by_level(tmp_path): quantiles = [{**PROPERTIES[5], "level": "0.5"}, {**PROPERTIES[5], "level": "0.25"}] result = _profile(tmp_path, _rows(), quantiles) - named = result.statistics + result.withheld_statistics + named = result.tables[0].statistics + result.tables[0].withheld_statistics shapes = {each[3] for each in named if each[1] == "quantile/1"} assert shapes == {"at level 0.5", "at level 0.25"} @@ -1045,9 +1045,9 @@ def test_a_statistic_with_some_groups_withheld_is_named_with_its_counts(tmp_path assert 0 < published < len(statuses) assert set(statuses) == {"F", "C"} named = ("amount", "quantile/2", "market", "at level 0.5 over bins 0, 50, 100") - assert result.partly_withheld_statistics == ((*named, published, len(statuses)),) - assert named not in result.statistics - assert named not in result.withheld_statistics + assert result.tables[0].partly_withheld_statistics == ((*named, published, len(statuses)),) + assert named not in result.tables[0].statistics + assert named not in result.tables[0].withheld_statistics def test_the_report_names_the_statistics_withheld_in_part(tmp_path, monkeypatch, capsys): @@ -1173,7 +1173,11 @@ def test_a_histogram_group_with_no_cell_left_is_withheld_whole(tmp_path): groups = {group["group"]: group for group in entry["groups"]} assert groups["DE"] == {"group": "DE", "status": "C"} assert {key for key, group in groups.items() if group["status"] == "F"} == {"FR", "GB", "IE"} - (named,) = [one for one in result.partly_withheld_statistics if one[1] == "marginalHistogram/1"] + (named,) = [ + one + for one in result.tables[0].partly_withheld_statistics + if one[1] == "marginalHistogram/1" + ] assert named[-2:] == (3, 4) load_profile_text(result.canonical.decode("utf-8")) diff --git a/tests/profiler/test_disclosure_oracle.py b/tests/profiler/test_disclosure_oracle.py index 445bf4cf..2f6a0dd9 100644 --- a/tests/profiler/test_disclosure_oracle.py +++ b/tests/profiler/test_disclosure_oracle.py @@ -3,8 +3,8 @@ """An exhaustive audit of the count tables a profile publishes: no small count is pinned. -For tiny seeded instances (one or two columns, at most three bins, at most seven rows) -this enumerates every world, every multiset of records, runs the real profiler on each +For tiny seeded instances (one or two tables, at most three bins, at most seven rows +or six parents) this enumerates every world, every multiset of records, runs the real profiler on each (the registry, the output checks and the document, through ``profile_exposes``), and groups the worlds by the document they publish. Worlds that publish the same document are the worlds a reader cannot tell apart, the publish decision included: a withheld @@ -30,14 +30,24 @@ ones, so a number column the contract's block bins withholds them and forms no ``mean/1``; - a histogram and quantiles on the same bins, checked together, let a reader who knows - the check's choices tell worlds apart, so a column gets one or the other. + the check's choices tell worlds apart, so a column gets one or the other; +- children per parent: the check sees the parents' cells and their total, while a + reader can know far more: other tables over the same parents bound their number, and + the children's own counts bound what they hold (each child row has at most one parent, + and a parent holds between its cell's edges). So the profiler audits every withheld + cell against the strongest reader, who knows the numbers of parents and of children + exactly, and withholds more until no cell can be worked out + (:func:`test_the_strongest_reader_pins_no_small_count_either` keys worlds by those + numbers too; :func:`test_children_per_parent_under_acro_pin_no_small_count` splits a + band of children over a band of parents every way, under ACRO's rule). """ from __future__ import annotations import itertools +import json from collections import defaultdict -from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple import pytest @@ -63,6 +73,8 @@ ) for cell, base in ((2, 1), (3, 1), (3, 2), (4, 2)) ] +#: The rules that round, as every profile's counts are rounded (ACRO rounds to 5). +ROUNDING_RULES = [rule for rule in RULES if rule.rounding_base > 1] Row = Tuple[Optional[str], ...] #: A protected count: its name, and the function that counts it in a world. @@ -112,6 +124,7 @@ def _without_new_statistics(document: Mapping[str, Any]) -> str: stripped = json.loads(json.dumps(document)) stripped.pop("_schemaURL", None) + stripped.pop("relationships", None) for table in stripped["tables"].values(): for column in table["columns"].values(): kept = [s for s in column.get("statistics", []) if s["definition"] == "mean/1"] @@ -253,3 +266,238 @@ def test_quantiles_on_a_types_ladder_pin_no_small_count(monkeypatch, rule): ran, pins = _audit(monkeypatch, contract, ["v"], values, _spans("v", cells), rule) assert ran == 330 assert pins == [] + + +#: Two tables: parents, and children whose key references them. +RELATIONSHIP_CONTRACT: Mapping[str, Any] = { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "oracle", + "exposes": [ + {"exposeId": "org", "contract": {"schema": [{"name": "key", "type": "string"}]}}, + { + "exposeId": "acct", + "contract": {"schema": [{"name": "org", "type": "string"}]}, + "semantics": { + "relationships": [ + { + "name": "owner", + "to": "org", + "fromColumns": ["org"], + "toColumns": ["key"], + "cardinality": "many_to_one", + } + ] + }, + }, + ], +} + + +def _read_parents_and_children(monkeypatch: pytest.MonkeyPatch, current: Dict[str, Any]) -> None: + """Have every read return ``current["counts"]``'s parents and their children.""" + + def read_columns(source, names, *, table=None, cwd=None): + counts = current["counts"] + if source == "org": + values = [f"p{n}" for n in range(len(counts))] + return sources.SourceColumns("org", len(values), {"key": values}) + values = [f"p{n}" for n, count in enumerate(counts) for _ in range(count)] + return sources.SourceColumns("acct", len(values), {"org": values}) + + monkeypatch.setattr(sources, "read_columns", read_columns) + + +def _profile_parents(rule: disclosure.DisclosureRule, backend: Any) -> Any: + return runner.profile_exposes( + RELATIONSHIP_CONTRACT, + [runner.ExposeSource("org", "org"), runner.ExposeSource("acct", "acct")], + backend=backend, + rule=rule, + ) + + +def _relationship_pins( + monkeypatch: pytest.MonkeyPatch, rules: Sequence[disclosure.DisclosureRule] +) -> List[str]: + """Two tables: parents, and children that reference them, 0 to 2 per parent. + + A world is the parents' count and the multiset of their children's counts; which parent + has which count is not seen by any statistic of string keys. The protected counts are + the parents with none, one and two children, and every run of them. + """ + backend = load_backend() + worlds = [ + counts + for parents in range(7) + for counts in itertools.combinations_with_replacement((0, 1, 2), parents) + ] + current: Dict[str, Any] = {} + _read_parents_and_children(monkeypatch, current) + quantities = [ + (f"parents with {low} to {high - 1} children", low, high) + for low in range(3) + for high in range(low + 1, 4) + ] + pins: List[str] = [] + for rule in rules: + groups: Dict[bytes, List[Tuple[int, ...]]] = defaultdict(list) + old: Dict[str, List[Tuple[int, ...]]] = defaultdict(list) + views: Dict[Tuple[int, ...], str] = {} + for counts in worlds: + current["counts"] = counts + result = _profile_parents(rule, backend) + assert result.relationships, "the relationship is counted" + groups[result.canonical].append(counts) + views[counts] = _without_new_statistics(result.document) + old[views[counts]].append(counts) + for group in groups.values(): + if any(len(counts) > 6 - rule.rounding_base for counts in group): + continue + for name, low, high in quantities: + seen = {sum(1 for c in counts if low <= c < high) for counts in group} + if len(seen) != 1: + continue + (value,) = seen + if value >= rule.minimum_cell or (value == 0 and not rule.zeros_disclosive): + continue + before = { + sum(1 for c in counts if low <= c < high) for counts in old[views[group[0]]] + } + if len(before) > 1: + pins.append(f"{rule.name}: {name} = {value} in worlds like {group[0]}") + assert len(worlds) == 84 + return pins + + +def test_children_per_parent_pin_no_small_count(monkeypatch): + assert _relationship_pins(monkeypatch, ROUNDING_RULES) == [] + + +def test_with_exact_counts_children_per_parent_pins_no_small_count(monkeypatch): + # The audit's reader knows the children's total exactly, as exact counts publish it. + exact = [rule for rule in RULES if rule.rounding_base == 1] + assert _relationship_pins(monkeypatch, exact) == [] + + +def _splits(total: int, parts: int, largest: Optional[int] = None) -> Iterable[Tuple[int, ...]]: + """Every multiset of at most *parts* positive counts adding up to *total*.""" + largest = total if largest is None else largest + if total == 0: + yield () + return + if parts == 0: + return + for first in range(min(total, largest), 0, -1): + for rest in _splits(total - first, parts - 1, first): + yield (first, *rest) + + +def _children_pins( + monkeypatch: pytest.MonkeyPatch, + worlds: Sequence[Tuple[int, ...]], + rule: disclosure.DisclosureRule, + interior: Callable[[Tuple[int, ...]], bool] = lambda world: True, + strongest: bool = False, +) -> List[str]: + """The protected counts of children per parent that *worlds* pin under *rule*. + + A world is each parent's number of children, none an orphan. The protected counts are + the histogram's cells and every run of them; a group holding a world off the + universe's *interior* is not judged, since worlds past its edge could join it. The + *strongest* reader also knows the number of parents and of children exactly, as other + tables over the same rows can tell them: worlds are grouped by those too. + """ + backend = load_backend() + current: Dict[str, Any] = {} + _read_parents_and_children(monkeypatch, current) + lower = runner.CHILDREN_EDGES[:-1] + + def cells(world: Sequence[int]) -> List[int]: + found = [0] * len(lower) + for count in world: + found[max(index for index, edge in enumerate(lower) if count >= edge)] += 1 + return found + + groups: Dict[Tuple[Any, ...], List[Tuple[int, ...]]] = defaultdict(list) + old: Dict[str, List[Tuple[int, ...]]] = defaultdict(list) + views: Dict[Tuple[int, ...], str] = {} + for world in worlds: + current["counts"] = world + result = _profile_parents(rule, backend) + known = (len(world), sum(world)) if strongest else () + groups[(result.canonical, *known)].append(world) + views[world] = json.dumps([_without_new_statistics(result.document), *known]) + old[views[world]].append(world) + pins: List[str] = [] + for group in groups.values(): + if not all(interior(world) for world in group): + continue + for low in range(len(lower)): + for high in range(low + 1, len(lower) + 1): + seen = {sum(cells(world)[low:high]) for world in group} + (value,) = seen if len(seen) == 1 else (None,) + if value is None or value >= rule.minimum_cell: + continue + if value == 0 and not rule.zeros_disclosive: + continue + before = {sum(cells(world)[low:high]) for world in old[views[group[0]]]} + if len(before) > 1: + pins.append( + f"{rule.name}: cells {low} to {high - 1} = {value}, as in {group[0]}" + ) + return pins + + +def _split_worlds(parents: Tuple[int, int], children: Tuple[int, int]) -> List[Tuple[int, ...]]: + """Every split of every total of *children* over every count of *parents*.""" + return [ + tuple(sorted((0,) * (count - len(split)) + split)) + for count in range(parents[0], parents[1] + 1) + for total in range(children[0], children[1] + 1) + for split in _splits(total, count) + ] + + +#: A band of parents and a band of children, which ACRO's rule rounds to 25 and 10, and +#: to 30 and 15: each band whole, so every world a document fits is enumerated. +UNDER_ACRO = [((23, 27), (10, 12), 875), ((28, 32), (13, 17), 4700)] + + +@pytest.mark.parametrize("parents, children, worlds", UNDER_ACRO) +def test_children_per_parent_under_acro_pin_no_small_count(monkeypatch, parents, children, worlds): + split = _split_worlds(parents, children) + assert len(split) == worlds + assert _children_pins(monkeypatch, split, disclosure.ACRO_DEFAULT) == [] + + +def test_without_the_audit_the_childrens_counts_pin_a_zero(monkeypatch): + # The output check alone sees the parents' cells and their total, not the children's. + monkeypatch.setattr(runner, "_audited_children", lambda decision, *rest: decision) + split = _split_worlds((23, 27), (10, 12)) + assert _children_pins(monkeypatch, split, disclosure.ACRO_DEFAULT) + + +@pytest.mark.parametrize("rule", RULES, ids=lambda rule: rule.name) +def test_the_strongest_reader_pins_no_small_count_either(monkeypatch, rule): + # The number of parents and of children known exactly, every rule, exact counts too. + split = _split_worlds((0, 7), (0, 14)) + + def interior(world: Tuple[int, ...]) -> bool: + return len(world) <= 7 - rule.rounding_base and sum(world) <= 14 - rule.rounding_base + + assert _children_pins(monkeypatch, split, rule, interior, strongest=True) == [] + + +@pytest.mark.parametrize("rule", ROUNDING_RULES, ids=lambda rule: rule.name) +def test_children_per_parent_with_unbounded_children_pin_no_small_count(monkeypatch, rule): + # Up to 7 parents and 14 children split every way: under these small rules the check + # publishes cells in many worlds and not in others, so its choices and the audit's + # are part of what a reader sees. + split = _split_worlds((0, 7), (0, 14)) + assert len(split) == 1613 + + def interior(world: Tuple[int, ...]) -> bool: + return len(world) <= 7 - rule.rounding_base and sum(world) <= 14 - rule.rounding_base + + assert _children_pins(monkeypatch, split, rule, interior) == [] diff --git a/tests/profiler/test_flux_stats_backend.py b/tests/profiler/test_flux_stats_backend.py index 3633c3ad..72ea9c97 100644 --- a/tests/profiler/test_flux_stats_backend.py +++ b/tests/profiler/test_flux_stats_backend.py @@ -234,7 +234,7 @@ def test_a_boolean_column_is_left_out_as_flux_stats_reads_none(tmp_path, backend } rows = [{"attempts": n % 4, "paid": n % 2 == 0} for n in range(40)] result = profile_expose(contract, "payment", source=_slice(tmp_path, rows), backend=backend) - assert [name for name, _ in result.plan.skipped] == ["paid"] + assert [name for name, _ in result.tables[0].plan.skipped] == ["paid"] load_profile_text(result.canonical.decode("utf-8")) diff --git a/tests/profiler/test_relationships.py b/tests/profiler/test_relationships.py new file mode 100644 index 00000000..a0e27d82 --- /dev/null +++ b/tests/profiler/test_relationships.py @@ -0,0 +1,468 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Children per parent: a histogram with the parent as unit, for each declared relationship.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fluid_build.profiler import disclosure, runner +from fluid_build.profiler.runner import CHILDREN_EDGES, ExposeSource, profile_exposes +from tests.profiler._fake_backend import FakeBackend + +pytestmark = pytest.mark.unit + +pytest.importorskip("flux_stats.fidelity") + + +def _contract(child_key="int", relationship=None, validity=False): + org = {"exposeId": "org", "contract": {"schema": [{"name": "org_id", "type": "int"}]}} + if validity: + org["semantics"] = { + "dimensions": [ + { + "name": "from", + "type": "time", + "typeParams": {"validityParams": {"isStart": True}}, + } + ] + } + account = { + "exposeId": "account", + "contract": { + "schema": [{"name": "account_id", "type": "int"}, {"name": "org_id", "type": child_key}] + }, + "semantics": { + "relationships": [ + relationship + or { + "name": "owner", + "to": "org", + "fromColumns": ["org_id"], + "toColumns": ["org_id"], + "cardinality": "many_to_one", + } + ] + }, + } + return {"fluidVersion": "0.7.7", "kind": "DataProduct", "id": "p", "exposes": [org, account]} + + +def _write(path: Path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return str(path) + + +def _children(counts): + """Child rows giving parent n the number of children ``counts[n]``.""" + rows, next_id = [], 0 + for parent, count in enumerate(counts): + for _ in range(count): + rows.append({"account_id": str(next_id), "org_id": str(parent)}) + next_id += 1 + return rows + + +def _profile(tmp_path, contract, counts, child_rows=None, parents=None): + org = _write( + tmp_path / "org.ndjson", parents or [{"org_id": str(n)} for n in range(len(counts))] + ) + account = _write(tmp_path / "account.ndjson", child_rows or _children(counts)) + return profile_exposes( + contract, + [ExposeSource("org", org), ExposeSource("account", account)], + backend=FakeBackend(), + ) + + +def test_each_parent_counts_once_by_its_number_of_children(tmp_path): + # 20 parents with none, 30 with one, 20 with two, 10 with four, 10 with 25. + counts = [0] * 20 + [1] * 30 + [2] * 20 + [4] * 10 + [25] * 10 + result = _profile(tmp_path, _contract(), counts) + (entry,) = result.document["relationships"] + assert entry["parent"] == {"table": "org", "columns": ["org_id"]} + assert entry["child"] == {"table": "account", "columns": ["org_id"]} + (statistic,) = entry["statistics"] + assert statistic["definition"] == "marginalHistogram/1" + assert statistic["bins"] == [str(edge) for edge in CHILDREN_EDGES] + assert statistic["bins"][-1] == "9007199254740991" # 20 or more, never cut + (group,) = statistic["groups"] + assert group["k"] == 90 # the parents, rounded as the parent's rowCount is + # 0, 1, 2, 3-4, 5-9, 10-19, 20+: the empty 5-9 and 10-19 are withheld, and so is a + # neighbour, or the total would give them back. + assert group["status"] == "F" + assert group["binCounts"][:3] == [20, 30, 20] + assert group["cellStatus"][4:6] == ["C", "C"] + (counted,) = result.relationships + assert counted.relationship.name == "owner" and counted.cells == 7 + + +def test_the_foreign_keys_distinct_count_is_withheld_beside_it(tmp_path): + counts = [0] * 20 + [1] * 30 + [2] * 20 + result = _profile(tmp_path, _contract(), counts) + metrics = result.document["tables"]["account"]["columns"]["org_id"]["metrics"] + # The parents with a child: beside the parents' rowCount, it would count those with none. + assert metrics["distinctCount"] == { + "definition": "distinctCount/1", + "status": "D", + "sampleSize": metrics["count"]["sampleSize"], + "disclosure": "acro-default", + } + # Its sum, extremes and quantiles describe each parent's children key by key, and it + # forms no mean/1. + for name in ("sum", "min", "max", "quantiles"): + assert metrics[name]["status"] == "D" + assert not result.document["tables"]["account"]["columns"]["org_id"].get("statistics") + # The parent's own key keeps it. + assert ( + result.document["tables"]["org"]["columns"]["org_id"]["metrics"]["distinctCount"]["status"] + == "F" + ) + + +def test_a_null_or_unmatched_key_is_no_parents_child(tmp_path, monkeypatch): + counted = [] + audit = runner._audited_children + + def spy(decision, group, *rest): + counted.append(group.cells) + return audit(decision, group, *rest) + + monkeypatch.setattr(runner, "_audited_children", spy) + counts = [1] * 20 + [2] * 20 + rows = _children(counts) + [ + {"account_id": "1001", "org_id": None}, # no parent + {"account_id": "1002", "org_id": "999"}, # a parent the table does not hold + ] + parents = [{"org_id": str(n)} for n in range(40)] + [{"org_id": None}] * 10 + result = _profile(tmp_path, _contract(), counts, child_rows=rows, parents=parents) + ((group,),) = [entry["statistics"][0]["groups"] for entry in result.document["relationships"]] + # The ten parents whose key is null have no child, 20 have one and 20 two; the two + # children that match no parent are counted for none. + assert counted == [(10, 20, 20, 0, 0, 0, 0)] + assert group["k"] == 50 + # Beside about 60 children, the parents with one child would show that none has 20 + # or more, so the audit withholds that cell too. + assert group["binCounts"][:2] == [10, None] + + +LOWER = CHILDREN_EDGES[:-1] + + +def test_the_audit_finds_a_high_zero_the_childrens_counts_give_back(): + # 503 parents: 100 with no child, 400 with one, 3 with two, so 406 children. With only + # the cell of no child published, the other 403 parents hold 406 children: none can + # hold 10, whatever a reader moves. + cells = (100, 400, 3, 0, 0, 0, 0) + pinned = disclosure.pinned_children_cells( + LOWER, cells, [True] + [False] * 6, 406, disclosure.ACRO_DEFAULT + ) + assert {5, 6} <= pinned + + +def test_a_cell_with_room_to_move_is_not_pinned(): + # 60 parents: 20 with no child, 20 with one, 20 with 25. A reader who knows the 520 + # children exactly can still move children between the parents of the top cell. + cells = (20, 20, 0, 0, 0, 0, 20) + published = [True, True, False, False, False, False, False] + pinned = disclosure.pinned_children_cells(LOWER, cells, published, 520, disclosure.ACRO_DEFAULT) + assert pinned == set() + + +def test_the_childrens_total_bounds_the_cells_from_below_too(): + # One child among 30 parents: a parent must hold it. With every other cell published, + # the cell of no child cannot gain the one parent that holds it. + cells = (29, 1, 0, 0, 0, 0, 0) + published = [False] + [True] * 6 + pinned = disclosure.pinned_children_cells(LOWER, cells, published, 1, disclosure.ACRO_DEFAULT) + assert pinned == {0} + + +def test_the_check_never_leaves_a_small_total_withheld(): + # The one withheld cell holds 10 parents: the check would have withheld more had it + # held fewer, so a reader knows it cannot have lost a parent to another cell. + cells = (10, 0, 0, 10, 0, 0, 0) + published = [True, True, True, False, True, True, True] + pinned = disclosure.pinned_children_cells(LOWER, cells, published, 35, disclosure.ACRO_DEFAULT) + assert pinned == {3} + + +def test_a_table_the_readers_model_does_not_hold_claims_nothing_safe(): + # 12 parents with a null key cannot all sit in a cell of 11 parents: every withheld + # cell is returned, so the audit withholds the histogram whole. + cells = (11, 12, 0, 0, 0, 0, 0) + pinned = disclosure.pinned_children_cells( + LOWER, cells, [True] + [False] * 6, 12, disclosure.ACRO_DEFAULT, childless=12 + ) + assert pinned == set(range(1, 7)) + + +@pytest.mark.parametrize( + "counts", + [[0] * 100 + [1] * 400 + [2] * 3, [0] * 12 + [1] * 11], + ids=["503 parents", "23 parents"], +) +def test_no_withheld_cell_follows_from_the_childrens_counts(tmp_path, counts): + result = _profile(tmp_path, _contract(), counts) + ((group,),) = [entry["statistics"][0]["groups"] for entry in result.document["relationships"]] + # The check published the cell of no child; beside the children's counts it pinned a + # high bin's zero, so the histogram is withheld whole. + assert group["status"] == "C" + + +def test_an_enum_of_the_parents_does_not_give_a_withheld_cell_back(tmp_path): + # The parents' level histogram bounds their number (10 + 13 + 18, and at least 10 + # withheld), tighter than their row count: beside the 41 children, no parent can hold + # 20. The strongest reader knows that number exactly, so the histogram is withheld. + counts = [0] * 27 + [1] * 14 + [2] * 6 + [3] * 5 + tiers = ["a"] * 10 + ["b"] * 13 + ["c"] * 18 + ["d"] * 2 + ["e"] * 9 + contract = _contract() + contract["exposes"][0]["contract"]["schema"].append( + {"name": "tier", "type": "enum('a', 'b', 'c', 'd', 'e')"} + ) + parents = [{"org_id": str(n), "tier": tier} for n, tier in enumerate(tiers)] + result = _profile(tmp_path, contract, counts, parents=parents) + tier = result.document["tables"]["org"]["columns"]["tier"]["statistics"][0]["groups"][0] + assert tier["levelCounts"][:3] == [10, 15, 20] + ((group,),) = [entry["statistics"][0]["groups"] for entry in result.document["relationships"]] + assert group["status"] == "C" + + +def test_two_relationships_into_one_parent_do_not_give_each_other_away(tmp_path): + # Each relationship's cells bound the parents' number for the other one. + first = [0] * 27 + [1] * 14 + [2] * 6 + [3] * 5 + second = [0] * 10 + [1] * 13 + [2] * 18 + [3] * 2 + [5] * 9 + relationship = { + "name": "owner", + "to": "org", + "fromColumns": ["org_id"], + "toColumns": ["org_id"], + "cardinality": "many_to_one", + } + contract = _contract() + contract["exposes"].append( + { + "exposeId": "invoice", + "contract": {"schema": [{"name": "org_id", "type": "int"}]}, + "semantics": {"relationships": [dict(relationship, name="billed")]}, + } + ) + org = _write(tmp_path / "org.ndjson", [{"org_id": str(n)} for n in range(52)]) + account = _write(tmp_path / "account.ndjson", _children(first)) + invoice = _write( + tmp_path / "invoice.ndjson", + [{"org_id": str(n)} for n, count in enumerate(second) for _ in range(count)], + ) + result = profile_exposes( + contract, + [ + ExposeSource("org", org), + ExposeSource("account", account), + ExposeSource("invoice", invoice), + ], + backend=FakeBackend(), + ) + groups = { + entry["child"]["table"]: entry["statistics"][0]["groups"][0] + for entry in result.document["relationships"] + } + assert groups["account"]["status"] == "C" + for name, counts in (("account", first), ("invoice", second)): + cells = [0] * len(LOWER) + for count in counts: + cells[max(i for i, edge in enumerate(LOWER) if count >= edge)] += 1 + group = groups[name] + if group["status"] == "F": # what it publishes leaves no withheld cell pinned + published = [status == "F" for status in group["cellStatus"]] + assert not disclosure.pinned_children_cells( + LOWER, cells, published, sum(counts), disclosure.ACRO_DEFAULT + ) + + +def test_a_relationship_whose_parent_is_not_profiled_is_named_and_left_out(tmp_path): + account = _write(tmp_path / "account.ndjson", _children([1] * 20)) + result = profile_exposes(_contract(), [ExposeSource("account", account)], backend=FakeBackend()) + assert "relationships" not in result.document + assert result.relationships_left_out == ( + ( + "account", + "owner", + "its target is not profiled in this call: profile both exposes together", + ), + ) + # No parent is counted, so the key's distinct count is not withheld for it. + metrics = result.document["tables"]["account"]["columns"]["org_id"]["metrics"] + assert metrics["distinctCount"]["status"] != "D" + + +def test_keys_read_in_different_domains_are_named_and_left_out(tmp_path): + result = _profile(tmp_path, _contract(child_key="string"), [1] * 20) + assert "relationships" not in result.document + ((child, name, reason),) = result.relationships_left_out + assert (child, name) == ("account", "owner") and "different domains" in reason + + +@pytest.mark.parametrize("columns", [5, "org_id", {"name": "org_id"}]) +def test_key_columns_that_are_not_lists_are_named_and_left_out(tmp_path, columns): + relationship = {"name": "owner", "to": "org", "fromColumns": columns, "toColumns": columns} + result = _profile(tmp_path, _contract(relationship=relationship), [1] * 20) + assert "relationships" not in result.document + ((_, name, reason),) = result.relationships_left_out + assert name == "owner" and "not two lists" in reason + + +def test_the_document_does_not_depend_on_the_order_the_exposes_are_named_in(tmp_path): + def expose(name, parent=None): + schema = [{"name": f"{name}_id", "type": "int"}] + entry = {"exposeId": name, "contract": {"schema": schema}} + if parent is not None: + key = f"{parent}_id" + schema.append({"name": key, "type": "int"}) + entry["semantics"] = { + "relationships": [ + {"name": f"of_{parent}", "to": parent, "fromColumns": [key], "toColumns": [key]} + ] + } + return entry + + contract = { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "p", + "exposes": [expose("org"), expose("account", "org"), expose("invoice", "account")], + } + sources = { + "org": _write(tmp_path / "org.ndjson", [{"org_id": str(n)} for n in range(20)]), + "account": _write( + tmp_path / "account.ndjson", + [{"account_id": str(n), "org_id": str(n % 20)} for n in range(40)], + ), + "invoice": _write( + tmp_path / "invoice.ndjson", + [{"invoice_id": str(n), "account_id": str(n % 40)} for n in range(80)], + ), + } + digests = set() + for order in (["org", "account", "invoice"], ["invoice", "account", "org"]): + result = profile_exposes( + contract, [ExposeSource(each, sources[each]) for each in order], backend=FakeBackend() + ) + children = [entry["child"]["table"] for entry in result.document["relationships"]] + assert children == ["account", "invoice"] # the contract's order + digests.add(result.digest) + assert len(digests) == 1 + + +def _edge_statistics(result, table, column): + statistics = result.document["tables"][table]["columns"][column].get("statistics", []) + return [each["definition"] for each in statistics if each["definition"] != "mean/1"] + + +def test_an_enum_key_takes_no_level_histogram_beside_children_per_parent(tmp_path): + # An enum key's level histogram counts each key's rows, and the child key's rows are + # each parent's children: the keys take none, and the relationship is counted. + keys = [f"L{n:02d}" for n in range(30)] + enum = "enum(" + ", ".join(f"'{key}'" for key in keys) + ")" + contract = _contract(child_key=enum) + contract["exposes"][0]["contract"]["schema"][0]["type"] = enum + counts = [0] * 10 + [12] * 20 + parents = [{"org_id": key} for key in keys] + rows = [ + {"account_id": str(n), "org_id": key} + for n, (key, count) in enumerate(zip(keys, counts, strict=True)) + for _ in range(count) + ] + result = _profile(tmp_path, contract, counts, child_rows=rows, parents=parents) + assert result.relationships and not result.relationships_left_out + assert _edge_statistics(result, "account", "org_id") == [] + assert _edge_statistics(result, "org", "org_id") == [] + uncovered = {name: why for table in result.tables for name, why in table.uncovered} + assert "counted relationship" in uncovered["org_id"] + + +@pytest.mark.parametrize( + "rules", [[], [{"type": "range", "constraint": "between 0 and 29"}]], ids=["ladder", "range"] +) +def test_a_number_key_takes_no_histogram_or_quantile(tmp_path, rules): + # A declared range would give each key a bin of its own, and the ladder's bins near 0 + # are one unit wide: either would count each parent's children beside the cells. + contract = _contract() + for expose in contract["exposes"]: + for column in expose["contract"]["schema"]: + if column["name"] == "org_id": + column["validationRules"] = rules + result = _profile(tmp_path, contract, [0] * 10 + [12] * 20) + assert result.relationships + assert _edge_statistics(result, "account", "org_id") == [] + assert _edge_statistics(result, "org", "org_id") == [] + # A column that is no key keeps its statistics. + assert _edge_statistics(result, "account", "account_id") != [] + + +def test_a_target_with_a_validity_window_is_left_out(tmp_path): + result = _profile(tmp_path, _contract(validity=True), [1] * 20) + assert "relationships" not in result.document + assert "validity window" in result.relationships_left_out[0][2] + + +def test_a_self_join_counts_each_row_as_a_parent(tmp_path): + contract = { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "p", + "exposes": [ + { + "exposeId": "employee", + "contract": { + "schema": [{"name": "id", "type": "int"}, {"name": "manager", "type": "int"}] + }, + "semantics": { + "relationships": [ + { + "name": "reports_to", + "to": "employee", + "fromColumns": ["manager"], + "toColumns": ["id"], + "cardinality": "many_to_one", + } + ] + }, + } + ], + } + # 10 managers with 3 reports each; the 30 reports and the managers manage nobody else. + rows = [{"id": str(m), "manager": None} for m in range(10)] + [ + {"id": str(100 + n), "manager": str(n % 10)} for n in range(30) + ] + source = _write(tmp_path / "employee.ndjson", rows) + result = profile_exposes(contract, [ExposeSource("employee", source)], backend=FakeBackend()) + ((group,),) = [entry["statistics"][0]["groups"] for entry in result.document["relationships"]] + assert group["k"] == 40 + assert group["binCounts"][0] == 30 # the reports have no one under them + assert result.relationships[0].relationship.parent == "employee" + + +def test_the_document_reads_back_with_its_relationship(tmp_path): + from flux_stats.profile import load_profile_text + + counts = [0] * 20 + [1] * 30 + [3] * 20 + result = _profile(tmp_path, _contract(), counts) + profile, _ = load_profile_text(result.canonical.decode("utf-8")) + (relationship,) = profile.relationships + assert (relationship.parent_table, relationship.child_table) == ("org", "account") + assert relationship.statistics[0].identifier == "marginalHistogram/1" + + +def test_the_summary_names_each_relationship(tmp_path, capsys): + from fluid_build.cli.profile_cmd import _report + + result = _profile(tmp_path, _contract(), [0] * 20 + [1] * 30 + [2] * 20) + lines = [] + _report(lambda text, **_: lines.append(text), result, tmp_path / "out.json") + text = " ".join(lines) + assert "Children per parent of exposes[org] in exposes[account] (owner):" in text + assert runner.RelationshipResult is type(result.relationships[0]) diff --git a/tests/profiler/test_runner.py b/tests/profiler/test_runner.py index 5eb6d3ac..1d4c770f 100644 --- a/tests/profiler/test_runner.py +++ b/tests/profiler/test_runner.py @@ -228,7 +228,7 @@ def test_a_number_column_carries_mean_1_on_the_documents_grid(tmp_path): "placed_at": ["quantile/2"], "note": [], } - assert ("amount", "mean/1", None, None) in result.statistics + assert ("amount", "mean/1", None, None) in result.tables[0].statistics assert result.fractional_bits == 16 @@ -365,7 +365,7 @@ def test_a_mean_is_withheld_where_the_sum_is(tmp_path, values, status): statistic, group = _mean(result) assert group == {"group": "", "status": status} assert statistic["sampleSize"] == {"min": 38, "max": 42} - assert not [each for each in result.statistics if each[1] == "mean/1"] + assert not [each for each in result.tables[0].statistics if each[1] == "mean/1"] assert "1234567" not in result.canonical.decode("utf-8") # no edge of the ladder either @@ -524,8 +524,6 @@ def test_several_exposes_go_into_one_document_each_from_its_own_source(tmp_path) assert tables["org"]["rowCount"] == 10 and tables["account"]["rowCount"] == 20 assert [table.plan.expose_id for table in result.tables] == ["account", "org"] assert result.digest == digest_of(result.canonical) - with pytest.raises(ValueError, match="one result per expose"): - result.plan # noqa: B018 - a profile of two exposes has no one plan def test_an_expose_named_twice_is_refused_before_any_read(tmp_path): @@ -546,7 +544,7 @@ def test_one_expose_profiles_as_it_always_did(tmp_path): ORDERS, [runner.ExposeSource("orders", _slice(tmp_path, rows))], backend=FakeBackend() ) assert one.canonical == many.canonical - assert one.plan == many.plan + assert one.tables == many.tables # -- a column the contract types date -----------------------------------------------------