diff --git a/AGENTS.md b/AGENTS.md index eae7965b..f8230c36 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,7 +164,7 @@ forge-cli/ | Bundle builder | `fluid_build/forge/core/bundle.py` — deterministic tgz + MANIFEST + fragment extraction + `$source` sentinel | | Build runners | `fluid_build/build_runners/{dbt,python}/` — extracted from legacy `cli/execute.py` in Phase 1 | | Masking at landing | `fluid_build/build_runners/_masking.py` — the DuckDB runner applies `exposes[].policy.privacy.masking[]` inside the COPY (file, S3, BigQuery staging, DLQ); secrets only from env (`FLUID_PII_HASH_SECRET`, `FLUID_PII_TOKENIZATION_KEY`, `FLUID_PII_ENCRYPTION_SECRET_KEY`, or `params.saltEnv`/`keyEnv`), held in UDF closures, never in SQL; an unappliable rule refuses the build, and so does a DuckDB without numpy (its `create_function` needs it; `[local]` names numpy, and the refusal names the extra). `cli/_verify_masking.py` is stage 9's shape check (local DuckDB, Athena in the count query) | -| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads each expose's records where they live, every expose of one call into one document (`runner.profile_exposes`; the CLI pairs `--source =` and `--table =` with repeated `--expose`) (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC document, format 1-0-0 or, once it holds a `quantile/2`, 1-1-0 (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). Without `--bundle` it declares, from the contract alone, a fidelity block of the statistics `fluxsim init` reads (`defaults.py`: a `levelHistogram/1` over every level of an `enum(...)` column, a `marginalHistogram/1` over a `validationRules` range of two ends inside the type (each value checked against the range exactly, `DeclaredRange`), else `quantile/2` on a ladder fixed by the type for numbers and dates, grouped level histograms per categorical `semantics.dimensions` level, at most `defaults.MOST_GROUPED_CELLS` cells per column; `DeclaredBlock.every_declared_group` lists every declared level as a group) and answers it through the same path; `tests/profiler/test_disclosure_oracle.py` enumerates every tiny world to show no new statistic pins a protected count, which is why a column never gets a histogram and quantiles together and why `runner._counted_levels` / `_binned_numbers` withhold `distinctCount`, and a declared range's `sum`, `min`, `max` and `quantiles`. With `--bundle DIR` it answers the fidelity block of DIR's one Simulation instead: `bundle.py` reads `*.flux.yml` as the engine does (flux-stats' `flux_stats.exact_yaml` and `flux_stats.canonical`, through the seam, one `ReadBudget` per bundle), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a quantile with bins is `quantile/2` (the profile then names format 1-1-0), the histograms and quantiles of one column go to flux-stats' `check_edge_counts` in one call (FLUX ADR 0012; `runner._edge_decisions`), a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | +| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads each expose's records where they live, every expose of one call into one document (`runner.profile_exposes`; the CLI pairs `--source =` and `--table =
` with repeated `--expose`) (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC document, format 1-0-0 or, once it holds a `quantile/2`, 1-1-0 (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). Without `--bundle` it declares, from the contract alone, a fidelity block of the statistics `fluxsim init` reads (`defaults.py`: a `levelHistogram/1` over every level of an `enum(...)` column, a `marginalHistogram/1` over a `validationRules` range of two ends inside the type (each value checked against the range exactly, `DeclaredRange`), else `quantile/2` on a ladder fixed by the type for numbers and dates, grouped level histograms per categorical `semantics.dimensions` level, at most `defaults.MOST_GROUPED_CELLS` cells per column; `DeclaredBlock.every_declared_group` lists every declared level as a group) and answers it through the same path; `tests/profiler/test_disclosure_oracle.py` enumerates every tiny world to show no new statistic pins a protected count, which is why a column never gets a histogram and quantiles together and why `runner._counted_levels` / `_binned_numbers` withhold `distinctCount`, and a declared range's `sum`, `min`, `max` and `quantiles`. A `semantics.relationships` entry between two exposes of one call gets children per parent in the document's `relationships[]` (`runner._children_per_parent`, bins `runner.CHILDREN_EDGES`, the parent as unit, its own `check_edge_counts` call), and the child's foreign key withholds its `distinctCount`, `sum`, `min`, `max` and `quantiles` and forms no `mean/1`; `runner._audited_children` then withholds every cell the strongest reader could work out: the numbers of parents, children and null-key parents known exactly, the check's own rule included (`disclosure.pinned_children_cells`, a witness search that errs toward withholding); the key columns of a counted relationship take no default statistic (`runner._relationship_keys`), since a key's own histogram or quantiles would count each parent's children. With `--bundle DIR` it answers the fidelity block of DIR's one Simulation instead: `bundle.py` reads `*.flux.yml` as the engine does (flux-stats' `flux_stats.exact_yaml` and `flux_stats.canonical`, through the seam, one `ReadBudget` per bundle), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a quantile with bins is `quantile/2` (the profile then names format 1-1-0), the histograms and quantiles of one column go to flux-stats' `check_edge_counts` in one call (FLUX ADR 0012; `runner._edge_decisions`), a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | | CI template generator | `fluid_build/forge/core/pipeline_templates.py` — 7-system generator; Jenkins ships the full 11-stage parameterized template with `--install-mode {pypi,dev-source}` | | Forge (AI creation) | `fluid_build/forge/` (templates, generators, extensions) | | Log redaction (global) | `fluid_build/observability/secret_redactor.py` — `SecretRedactingFilter` wired into Python logging | diff --git a/CHANGELOG.md b/CHANGELOG.md index f178da46..70eb66ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`fluid profile` counts children per parent for each declared relationship.** When one call + profiles both exposes of a `semantics.relationships` entry, without `--bundle`, the + document's `relationships[]` (PROFILE-SPEC §5, format unchanged) carries a + `marginalHistogram/1` with the parent as unit: each parent row counts once, by its number of + child rows, on the bins 0, 1, 2, 3 to 4, 5 to 9, 10 to 19, and 20 or more. The last bin ends + at 2^53 - 1, the largest count a profile holds, so no count is cut to fit. Keys are matched + as the registry reads them, and a null key matches none. The histogram goes through + `check_edge_counts` in a call of its own, as flux-stats documents for a relationship's count + of children. The child's foreign-key columns withhold their `distinctCount`, `sum`, `min`, + `max` and `quantiles` under `D` and form no `mean/1`: the parents that have a child, beside + the parents' row count, would give those with none, and the key's own values describe each + parent's children key by key. A relationship whose target is not profiled in the call, + declares a validity window, or whose keys are read in different domains is named in the + summary and left out. The key columns + of a counted relationship take no default statistic of their own: a key's level histogram, + histogram or quantiles count each key's rows, and the child key's rows are each parent's + children. The check sees the parents' cells and their total, while a reader can know more: + other tables over the same parents (an `enum` column's level histogram, another + relationship's cells) bound their number, and the children's own counts bound what they + hold, each child row having at most one parent and each parent between its bin's edges. + So each withheld cell is audited against the strongest reader, who knows the number of + parents, of children and of parents whose key is null exactly, besides the published + bands and the check's own rule: a withheld cell stands only where two tables that reader + can hold true differ in it. While one cannot be shown to, the published cell of the + highest edge is withheld too, down to the whole histogram, which is where parents with + few children often land. Audited world by world with children unbounded and none an + orphan: under ACRO's rule no split of a band of children over a band of parents pins a + count (up to 144,962 worlds), and up to 7 parents keyed by their exact numbers pin none + under every scaled rule, exact counts included. + ## [0.36.0] — 2026-11-19 ### Added diff --git a/fluid_build/cli/profile_cmd.py b/fluid_build/cli/profile_cmd.py index fe93f97e..35996448 100644 --- a/fluid_build/cli/profile_cmd.py +++ b/fluid_build/cli/profile_cmd.py @@ -68,7 +68,14 @@ date or a timestamp); - for each categorical semantics.dimensions entry on an enum column, the other enum columns' histograms per level of it, every declared level a - group, up to 10,000 cells for one column's groupings. + group, up to 10,000 cells for one column's groupings; + - for each semantics.relationships entry between two exposes of the call, + children per parent: how many child rows each parent row has, counted on + the bins 0, 1, 2, 3-4, 5-9, 10-19 and 20 or more, with the parent as the + unit; the child's foreign key then withholds its distinctCount, sum, + min, max and quantiles, its key columns take no statistic of their own, + and a cell a reader who knows the numbers of parents and children + exactly could work out is withheld too. A record outside what the contract declares (a level its type does not list, a value outside its type or its declared range) refuses the profile. @@ -734,6 +741,20 @@ def _report(say: Any, result: Any, out: Path, *, answered_block: bool = False) - ) for table in tables: _report_table(say, table, f"exposes[{table.plan.expose_id}]: " if several else "", grid) + for counted in result.relationships: + link = counted.relationship + shown = ( + f"{counted.published} of {counted.cells} cells published" + if counted.published + else "withheld whole" + ) + say( + f"Children per parent of exposes[{link.parent}] in exposes[{link.child}] " + f"({link.name}): {shown}.", + soft_wrap=True, + ) + for child, name, reason in result.relationships_left_out: + say(f"No children per parent for exposes[{child}] {name!r}: {reason}.", soft_wrap=True) if not result.published: say("Every value was withheld: the document says nothing about the records.") say(f"Wrote {out} (readable by you alone; read it before you share it).", soft_wrap=True) diff --git a/fluid_build/profiler/_stats.py b/fluid_build/profiler/_stats.py index 6c60137e..21a33c59 100644 --- a/fluid_build/profiler/_stats.py +++ b/fluid_build/profiler/_stats.py @@ -396,6 +396,15 @@ def measure( record the block's declaration refuses. """ + def count_histogram( + self, counts: Sequence[int], edges: Sequence[int], fractional_bits: int + ) -> MeasuredGroup: + """``marginalHistogram/1`` over whole numbers, one per unit: the ungrouped group. + + *counts* are one value per unit (a parent's number of children), and *edges* the + bins' whole-number edges, placed on the grid as the registry places any edge. + """ + def check_edge_counts( self, statistics: Sequence[EdgeStatistic], @@ -522,6 +531,7 @@ def __init__(self) -> None: from flux_stats.reading import ValueDomain, category, read_exact self._metrics = metrics + self._parameters = Parameters self._statistics = tuple(_Statistic(each, Parameters()) for each in definitions()) self._grid = Grid self._refused = FluxError @@ -823,6 +833,27 @@ def _measured( ) return replace(group, values=tuple(values), encoded=tuple(encoded)) + def count_histogram( + self, counts: Sequence[int], edges: Sequence[int], fractional_bits: int + ) -> MeasuredGroup: + grammar = self._fidelity() + grid = self._grid(fractional_bits) + histogram = grammar.definition_named("marginalHistogram/1") + encoded = tuple(grid.quantise(Fraction(edge), subject="bin edge") for edge in edges) + state = histogram.measure( + "", + [grid.quantise(Fraction(count), subject="count") for count in counts], + self._parameters(edges=encoded), + ) + spelled: Dict[str, Any] = state.to_document(histogram) + return MeasuredGroup( + key="", + document=spelled, + k=state.k, + statistic=state, + cells=tuple(state.cell_counts("marginalHistogram")), + ) + def check_edge_counts( self, statistics: Sequence[EdgeStatistic], diff --git a/fluid_build/profiler/disclosure.py b/fluid_build/profiler/disclosure.py index 44106fe7..2d3267eb 100644 --- a/fluid_build/profiler/disclosure.py +++ b/fluid_build/profiler/disclosure.py @@ -113,7 +113,19 @@ import unicodedata from dataclasses import dataclass from fractions import Fraction -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union +from typing import ( + Any, + Callable, + Dict, + FrozenSet, + Iterable, + List, + Optional, + Sequence, + Set, + Tuple, + Union, +) __all__ = [ "ACRO_DEFAULT", @@ -138,6 +150,7 @@ "check_rounding", "check_sum", "check_support", + "pinned_children_cells", "round_count", "sample_size", "suppress_secondary", @@ -533,3 +546,114 @@ def suppress_secondary(verdicts: Sequence[Verdict]) -> Tuple[Verdict, ...]: return tuple(verdicts) withheld = Verdict(SECONDARY, "another group of the statistic is withheld") return tuple(withheld if verdict.published else verdict for verdict in verdicts) + + +def pinned_children_cells( + lower_edges: Sequence[int], + cells: Sequence[int], + published: Sequence[bool], + children: int, + rule: DisclosureRule, + childless: int = 0, +) -> FrozenSet[int]: + """The withheld cells of a children-per-parent table a reader may work out exactly. + + The table counts parents by their number of children, on bins whose lower edges are + *lower_edges* (the first 0; each bin ends below the next, the last has no top). The + reader is the strongest a profile can meet: it knows the exact number of parents, + the exact number of children they hold together (*children*), how many parents hold + none because their key is null (*childless*), each published cell's band, and the + output check's own rule, so that where a cell is published the withheld cells + together are not a small count, nor a zero where zeros are disclosive. Any table it + can hold true has nonnegative cells with the parents' sum, ``lower * c <= children + <= upper * c`` (a parent holds between its bin's edges), at least *childless* parents + in the first cell, and each published cell in its band. + + A withheld cell is safe when two such tables differ in it. The true table (*cells*) + is one; the other is searched for among the tables one or two parents' moves away. + A cell with no such pair is returned, so the search errs toward withholding: a + returned cell may be one no reader could pin after all. + """ + count = len(cells) + tops: List[Optional[int]] = [edge - 1 for edge in lower_edges[1:]] + [None] + bands = [ + sample_size(value, rule) if shown else None + for value, shown in zip(cells, published, strict=True) + ] + any_published = any(published) + + def holds(table: Sequence[int]) -> bool: + if min(table) < 0 or table[0] < childless: + return False + for value, band in zip(table, bands, strict=True): + if band is not None and not band["min"] <= value <= band["max"]: + return False + if any_published: + rest = sum(value for value, shown in zip(table, published, strict=True) if not shown) + if not check_count(rest, rule).published: + return False + if sum(edge * value for edge, value in zip(lower_edges, table, strict=True)) > children: + return False + if tops[-1] is None and table[-1] > 0: + return True + return sum((top or 0) * value for top, value in zip(tops, table, strict=True)) >= children + + withheld = [index for index, shown in enumerate(published) if not shown] + if not holds(cells): + return frozenset(withheld) # the reader's model misses these records: no claim + moves = [ + (source, target) for source in range(count) for target in range(count) if source != target + ] + + def repairs(table: Sequence[int], source: int, target: int) -> Set[int]: + """One parent, or as many moved *source* to *target* as the children's bounds need.""" + many = {1} + low = sum(edge * value for edge, value in zip(lower_edges, table, strict=True)) + if low > children and lower_edges[source] > lower_edges[target]: + gap = lower_edges[source] - lower_edges[target] + many.add(-(-(low - children) // gap)) + unbounded = tops[-1] is None and table[-1] > 0 + if not unbounded: + high = sum((top or 0) * value for top, value in zip(tops, table, strict=True)) + if high < children and tops[target] is None: + many.add(1) + elif high < children and (tops[target] or 0) > (tops[source] or 0): + gap = (tops[target] or 0) - (tops[source] or 0) + many.add(-(-(children - high) // gap)) + return {each for each in many if 1 <= each <= table[source]} + + pinned: Set[int] = set() + for index in withheld: + if not _two_tables(cells, index, moves, holds, repairs): + pinned.add(index) + return frozenset(pinned) + + +def _two_tables( + cells: Sequence[int], + index: int, + moves: Sequence[Tuple[int, int]], + holds: Callable[[Sequence[int]], bool], + repairs: Callable[[Sequence[int], int, int], Set[int]], +) -> bool: + """Whether a table two moves from *cells* holds and differs at *index*. + + The first move takes one parent into or out of *index*; the second moves one parent, + or as many as the children's bounds then need (*repairs*), between any two cells. + """ + for source, target in moves: + if index not in (source, target): + continue + first = list(cells) + first[source] -= 1 + first[target] += 1 + if first[index] != cells[index] and holds(first): + return True + for again_source, again_target in moves: + for many in repairs(first, again_source, again_target): + second = list(first) + second[again_source] -= many + second[again_target] += many + if second[index] != cells[index] and holds(second): + return True + return False diff --git a/fluid_build/profiler/document.py b/fluid_build/profiler/document.py index f4c309a3..65aa5093 100644 --- a/fluid_build/profiler/document.py +++ b/fluid_build/profiler/document.py @@ -27,7 +27,7 @@ import hashlib import unicodedata -from typing import Any, Dict, List, Mapping +from typing import Any, Dict, List, Mapping, Sequence from fluid_build.profiler.disclosure import MAX_COUNT, DisclosureRule @@ -103,18 +103,26 @@ def build_document( rule: DisclosureRule, tables: Mapping[str, Mapping[str, Any]], fractional_bits: int = DEFAULT_FRACTIONAL_BITS, + relationships: Sequence[Mapping[str, Any]] = (), ) -> Dict[str, Any]: - """Assemble a document from tables already checked by *rule*, in the format they need.""" + """Assemble a document from tables already checked by *rule*, in the format they need. + + *relationships* are PROFILE-SPEC §5's children per parent, also checked by *rule*; + a document without any has no ``relationships`` key, as before there were any. + """ check_fractional_bits(fractional_bits) if not tables: raise DocumentError("tables holds at least one table") - return { + built: Dict[str, Any] = { "_producer": producer, "_schemaURL": schema_url_for(tables), "encoding": {"fractionalBits": fractional_bits, "rounding": "floor"}, "disclosureRules": {rule.name: rule.to_document()}, "tables": {expose_id: dict(table) for expose_id, table in tables.items()}, } + if relationships: + built["relationships"] = [dict(each) for each in relationships] + return built def _check(value: Any, where: List[str]) -> None: diff --git a/fluid_build/profiler/runner.py b/fluid_build/profiler/runner.py index 09609398..3e0ee1bf 100644 --- a/fluid_build/profiler/runner.py +++ b/fluid_build/profiler/runner.py @@ -39,7 +39,7 @@ from datetime import date from fractions import Fraction from pathlib import Path -from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Sequence, Set, Tuple from fluid_build.policy._common import iter_exposes from fluid_build.profiler import defaults, disclosure, document, sources @@ -64,9 +64,12 @@ "DEFAULT_QUANTILE_LEVELS", "ColumnPlan", "ExposePlan", + "CHILDREN_EDGES", "ExposeSource", "ProfileError", "ProfileResult", + "Relationship", + "RelationshipResult", "TableResult", "column_domain", "expose_ids", @@ -204,6 +207,33 @@ class TableResult: uncovered: Tuple[Tuple[str, str], ...] = () +#: The bins a parent's number of children is counted in: 0, 1, 2, 3 to 4, 5 to 9, 10 to 19, +#: and 20 or more. The last bin ends at the largest count a profile holds, 2^53 - 1, so it +#: holds every parent with 20 children or more and no count is cut to fit. +CHILDREN_EDGES: Tuple[int, ...] = (0, 1, 2, 3, 5, 10, 20, disclosure.MAX_COUNT) + + +@dataclass(frozen=True) +class Relationship: + """A declared relationship whose two exposes one profile reads: a child's foreign key.""" + + child: str + name: str + parent: str + child_columns: Tuple[str, ...] + parent_columns: Tuple[str, ...] + + +@dataclass(frozen=True) +class RelationshipResult: + """What the profile says about one relationship's children per parent, for the summary.""" + + relationship: Relationship + #: The cells published, of the histogram's cells; 0 when it is withheld whole. + published: int + cells: int + + @dataclass(frozen=True) class ProfileResult: document: Dict[str, Any] @@ -221,33 +251,10 @@ class ProfileResult: withheld_cells: Mapping[str, int] = field(default_factory=dict) #: The grid the document declares and its on-grid statistics were formed on. fractional_bits: int = DEFAULT_FRACTIONAL_BITS - - def _only(self) -> TableResult: - if len(self.tables) != 1: - raise ValueError("a profile of several exposes has one result per expose: read .tables") - return self.tables[0] - - # The one expose of a profile of one expose, as these were before several could be read. - - @property - def plan(self) -> ExposePlan: - return self._only().plan - - @property - def statistics(self) -> Tuple[NamedStatistic, ...]: - return self._only().statistics - - @property - def withheld_statistics(self) -> Tuple[NamedStatistic, ...]: - return self._only().withheld_statistics - - @property - def partly_withheld_statistics(self) -> Tuple[PartlyNamedStatistic, ...]: - return self._only().partly_withheld_statistics - - @property - def left_out(self) -> Tuple[Tuple[str, str], ...]: - return self._only().left_out + #: Each relationship counted, children per parent. + relationships: Tuple[RelationshipResult, ...] = () + #: Each declared relationship not counted: the child, the relationship, and why. + relationships_left_out: Tuple[Tuple[str, str, str], ...] = () def _base_type(type_name: Any) -> str: @@ -587,11 +594,20 @@ def profile_exposes( grids[plan.expose_id] = grid fractional_bits = _one_grid(grids, blocks, fractional_bits) uncovered: List[Tuple[Tuple[str, str], ...]] = [() for _ in plans] + relationships, relationships_left_out = _declared_relationships(contract, plans, blocks) + held = _relationship_keys(relationships) for position, plan in enumerate(plans): if plan.expose_id not in blocks: declared[position], uncovered[position] = _default_block( - contract, plan, backend, fractional_bits + contract, plan, backend, fractional_bits, held.get(plan.expose_id, frozenset()) ) + edges = { + plan.expose_id: _edge_counted(found) for plan, found in zip(plans, declared, strict=True) + } + relationships, relationships_left_out = _keys_not_counted_at_edges( + relationships, relationships_left_out, edges + ) + keys = _key_columns(relationships) reads = [ _dates_read( plan, @@ -613,9 +629,15 @@ def profile_exposes( rule, tallies, fractional_bits, + keys.get(plan.expose_id, frozenset()), ) tables_doc[plan.expose_id] = table_doc results.append(replace(result, uncovered=missing)) + by_expose = {plan.expose_id: (plan, read) for plan, read in zip(plans, reads, strict=True)} + counted = [ + _children_per_parent(each, by_expose, backend, rule, tallies, fractional_bits) + for each in relationships + ] if producer is None: from fluid_build import __version__ @@ -625,6 +647,7 @@ def profile_exposes( rule=rule, tables=tables_doc, fractional_bits=fractional_bits, + relationships=[entry for entry, _ in counted], ) canonical = document.canonical_bytes(built) backend.check_document(canonical.decode("utf-8")) @@ -639,6 +662,8 @@ def profile_exposes( withheld=dict(sorted(tally.items())), withheld_cells=dict(sorted(cell_tally.items())), fractional_bits=fractional_bits, + relationships=tuple(result for _, result in counted), + relationships_left_out=tuple(relationships_left_out), ) @@ -676,20 +701,303 @@ def _as_day(value: Any) -> Any: return value +def _declared_relationships( + contract: Mapping[str, Any], + plans: Sequence[ExposePlan], + blocks: Mapping[str, BundleBlock], +) -> Tuple[List[Relationship], List[Tuple[str, str, str]]]: + """The ``semantics.relationships`` this profile counts children per parent for. + + A relationship is counted when the profile reads both its exposes, without a bundle + (a bundle's profile answers its block alone), and both keys are columns it reads in + one domain each, pair by pair. One whose target declares a validity window is not: + a parent there has a row per version. Every other is returned with why, never + guessed at. Both come in the contract's order, whatever order the call names the + exposes in, so the document's bytes do not depend on it. + """ + from fluid_build.data_rules._model import declares_validity_window + + exposes = { + str(item.get("exposeId")): item + for item in iter_exposes(dict(contract)) + if item.get("exposeId") + } + planned = {plan.expose_id: plan for plan in plans} + counted: List[Relationship] = [] + left_out: List[Tuple[str, str, str]] = [] + for plan in (planned[expose_id] for expose_id in exposes if expose_id in planned): + semantics = exposes[plan.expose_id].get("semantics") + declared = semantics.get("relationships") if isinstance(semantics, Mapping) else None + for raw in declared if isinstance(declared, list) else (): + if not isinstance(raw, Mapping): + continue + name, target = str(raw.get("name") or "?"), raw.get("to") + child_columns = _column_names(raw.get("fromColumns")) + parent_columns = _column_names(raw.get("toColumns")) + reason: Optional[str] = None + if not isinstance(target, str) or target not in planned: + reason = "its target is not profiled in this call: profile both exposes together" + elif plan.expose_id in blocks or target in blocks: + reason = "a profile that answers a bundle answers its fidelity block alone" + elif declares_validity_window(exposes[target]): + reason = "its target declares a validity window, so a parent has a row per version" + elif not child_columns or len(child_columns) != len(parent_columns): + reason = "its fromColumns and toColumns are not two lists of one length" + else: + reason = _keys_unread(plan, planned[target], child_columns, parent_columns) + if reason is not None: + left_out.append((plan.expose_id, name, reason)) + else: + counted.append( + Relationship(plan.expose_id, name, str(target), child_columns, parent_columns) + ) + return counted, left_out + + +def _column_names(value: Any) -> Tuple[str, ...]: + """The column names a relationship lists; none unless *value* is a list.""" + return tuple(str(each) for each in value) if isinstance(value, list) else () + + +def _keys_unread( + child: ExposePlan, + parent: ExposePlan, + child_columns: Sequence[str], + parent_columns: Sequence[str], +) -> Optional[str]: + """Why two keys cannot be matched as read, or ``None`` when every pair can be.""" + child_domains = {column.name: column.domain for column in child.columns} + parent_domains = {column.name: column.domain for column in parent.columns} + for mine, theirs in zip(child_columns, parent_columns, strict=True): + if mine not in child_domains or theirs not in parent_domains: + return f"key column {mine!r} or {theirs!r} is not one fluid profile reads" + if child_domains[mine] != parent_domains[theirs]: + return f"key columns {mine!r} and {theirs!r} are read in different domains" + return None + + +def _key_columns(relationships: Sequence[Relationship]) -> Dict[str, FrozenSet[str]]: + """The foreign-key columns of each child: their ``distinctCount`` is withheld. + + A foreign key's distinct values are the parents that have a child, so beside the + parent's row count it would give the parents with none, a cell of the histogram. + """ + found: Dict[str, Set[str]] = {} + for each in relationships: + found.setdefault(each.child, set()).update(each.child_columns) + return {child: frozenset(columns) for child, columns in found.items()} + + +def _relationship_keys(relationships: Sequence[Relationship]) -> Dict[str, FrozenSet[str]]: + """The key columns of every counted relationship, both sides, by expose. + + They take no default statistic: a key's own histogram or quantile counts each key's + rows, and the child key's rows are each parent's children, so beside children per + parent it would give a parent's count back. Their children per parent describe them. + """ + found: Dict[str, Set[str]] = {} + for each in relationships: + found.setdefault(each.child, set()).update(each.child_columns) + found.setdefault(each.parent, set()).update(each.parent_columns) + return {expose: frozenset(columns) for expose, columns in found.items()} + + +def _edge_counted(declared: Optional[DeclaredBlock]) -> FrozenSet[str]: + """The columns a statistic counts records at the edges of, or groups by.""" + if declared is None: + return frozenset() + found = set() + for statistic in declared.statistics: + if statistic.kind in _EDGE_KINDS: + found.add(statistic.column) + if statistic.group_by is not None: + found.add(statistic.group_by) + return frozenset(found) + + +def _keys_not_counted_at_edges( + relationships: Sequence[Relationship], + left_out: Sequence[Tuple[str, str, str]], + edges: Mapping[str, FrozenSet[str]], +) -> Tuple[List[Relationship], List[Tuple[str, str, str]]]: + """*relationships* less any whose key a statistic still counts at its edges. + + The default blocks leave every key out, so none should be; one that is would give + each parent's children beside the histogram, and is named and left out instead. + """ + kept: List[Relationship] = [] + out = list(left_out) + for each in relationships: + if set(each.child_columns) & edges.get(each.child, frozenset()) or set( + each.parent_columns + ) & edges.get(each.parent, frozenset()): + out.append((each.child, each.name, "a key column is counted at its edges")) + else: + kept.append(each) + return kept, out + + +def _children_per_parent( + relationship: Relationship, + tables: Mapping[str, Tuple[ExposePlan, sources.SourceColumns]], + backend: StatisticsBackend, + rule: disclosure.DisclosureRule, + tallies: Tuple["Counter[str]", "Counter[str]"], + fractional_bits: int, +) -> Tuple[Dict[str, Any], RelationshipResult]: + """The ``relationships[]`` entry of one relationship: a histogram with the parent as unit. + + Each parent row counts once, by the child rows whose key equals its own, compared as + the registry reads both (a parent whose key is null, like a child's null key, matches + none). The counts go through the output check for counts at edges in a call of their + own, as ``check_edge_counts`` documents for a relationship's count of children, and + are spelled as a column's histogram is. + """ + parents = _keys(tables[relationship.parent], relationship.parent_columns, backend) + children = Counter( + key + for key in _keys(tables[relationship.child], relationship.child_columns, backend) + if None not in key + ) + counts = [0 if None in key else children.get(key, 0) for key in parents] + encode = backend.encoder(fractional_bits) + statistic = DeclaredStatistic( + identifier="marginalHistogram/1", + kind="marginalHistogram", + column=f"{relationship.child}.{relationship.name}", + group_by=None, + shape={"bins": [str(edge) for edge in CHILDREN_EDGES]}, + columns=(), + read_as={}, + edges=tuple(encode(Fraction(edge)) for edge in CHILDREN_EDGES), + ) + measured = MeasuredStatistic( + groups=(backend.count_histogram(counts, CHILDREN_EDGES, fractional_bits),) + ) + try: + (decision,) = backend.check_edge_counts( + [_edge_statistic(statistic, measured, rule)], + total=_column_total(statistic, measured), + minimum_cell=rule.minimum_cell, + zeros_disclosive=rule.zeros_disclosive, + round_count=lambda count: disclosure.round_count(count, rule), + ) + except ValueError as exc: + raise ProfileError( + "profile_edge_counts_refused", + f"the children per parent of {statistic.column} cannot describe one set of " + f"parents: {exc}", + {"written": "nothing", "next": "report this: the output check refused its input"}, + ) from None + childless = sum(1 for key in parents if None in key) + decision = _audited_children(decision, measured.groups[0], sum(counts), childless, rule) + entry = _declared_entry(statistic, measured, rule, tallies, fractional_bits, decision) + (group,) = entry["groups"] + published = ( + sum(1 for count in group["binCounts"] if count is not None) + if group["status"] == disclosure.FREE + else 0 + ) + document_entry = { + "parent": {"table": relationship.parent, "columns": list(relationship.parent_columns)}, + "child": {"table": relationship.child, "columns": list(relationship.child_columns)}, + "statistics": [entry], + } + return document_entry, RelationshipResult( + relationship, published=published, cells=len(CHILDREN_EDGES) - 1 + ) + + +#: Why the audit of children per parent withholds a cell the output check let through. +_CHILDREN_PINNED = "the parents' and the children's counts would give a withheld cell back" + + +def _audited_children( + decision: EdgeDecision, + group: MeasuredGroup, + children: int, + childless: int, + rule: disclosure.DisclosureRule, +) -> EdgeDecision: + """*decision*, with more cells withheld until no withheld cell can be worked out. + + ``check_edge_counts`` sees the parents' cells and their total. A reader can know more: + the other tables over the same parents (an enum's level histogram, another + relationship's cells) bound their number, and the children's own counts bound the + children they hold, each child row having at most one parent. So the audit takes the + strongest reader, who knows the parents' number, their *children* and the *childless* + ones whose key is null exactly (:func:`disclosure.pinned_children_cells`). While it + can pin a withheld cell, the published cell of the highest lower edge is withheld + too, down to every cell, which withholds the histogram whole. It reads counts the + profiler formed, never a record's value, and writes only which cells are withheld. + """ + if decision.group_status[group.key] != disclosure.FREE: + return decision + lower = CHILDREN_EDGES[:-1] + statuses = list(decision.cell_status[group.key]) + reasons = list(decision.cell_reason[group.key]) + while any(status == disclosure.FREE for status in statuses): + published = [status == disclosure.FREE for status in statuses] + if not disclosure.pinned_children_cells( + lower, group.cells, published, children, rule, childless + ): + break + heaviest = max( + (index for index, shown in enumerate(published) if shown), + key=lambda index: lower[index], + ) + statuses[heaviest] = disclosure.WITHHELD + reasons[heaviest] = _CHILDREN_PINNED + return replace( + decision, + cell_status={**decision.cell_status, group.key: tuple(statuses)}, + cell_reason={**decision.cell_reason, group.key: tuple(reasons)}, + ) + + +def _keys( + table: Tuple[ExposePlan, sources.SourceColumns], + names: Sequence[str], + backend: StatisticsBackend, +) -> List[Tuple[Any, ...]]: + """Each row's key over *names*, every part read as the registry reads its column.""" + plan, read = table + domains = {column.name: column.domain for column in plan.columns} + columns = [ + [ + backend.read_value(domains[name], value, column=name, row_index=index) + for index, value in enumerate(read.values[name]) + ] + for name in names + ] + return list(zip(*columns, strict=True)) if columns else [] + + def _default_block( - contract: Mapping[str, Any], plan: ExposePlan, backend: StatisticsBackend, fractional_bits: int + contract: Mapping[str, Any], + plan: ExposePlan, + backend: StatisticsBackend, + fractional_bits: int, + keys: FrozenSet[str] = frozenset(), ) -> Tuple[Optional[DeclaredBlock], Tuple[Tuple[str, str], ...]]: """The block the profiler declares for *plan* from the contract alone, read by the registry. ``(None, ...)`` when the contract declares nothing to answer. The block is the - profiler's own, so a grammar refusal of it is a defect, and says so. + profiler's own, so a grammar refusal of it is a defect, and says so. The *keys* of + counted relationships take no statistic in it (:func:`_relationship_keys`). """ expose = next( item for item in iter_exposes(dict(contract)) if item.get("exposeId") == plan.expose_id ) - planned = defaults.default_plan(expose, plan.columns, backend.enum_levels, fractional_bits) + columns = [column for column in plan.columns if column.name not in keys] + held = tuple( + (column.name, "a key of a counted relationship, which its children per parent describe") + for column in plan.columns + if column.name in keys + ) + planned = defaults.default_plan(expose, columns, backend.enum_levels, fractional_bits) if planned.block is None: - return None, planned.uncovered + return None, planned.uncovered + held try: declared = backend.read_block(planned.block, contract, plan.expose_id) except BlockRefused as exc: @@ -704,7 +1012,7 @@ def _default_block( ) from None _check_declared_columns(declared, plan) block = replace(declared, every_declared_group=True, declared_ranges=planned.ranges) - return block, planned.uncovered + return block, planned.uncovered + held def _refuse_repeated(exposes: Sequence[ExposeSource]) -> None: @@ -765,12 +1073,18 @@ def _profile_table( rule: disclosure.DisclosureRule, tallies: Tuple["Counter[str]", "Counter[str]"], fractional_bits: int, + keys: FrozenSet[str] = frozenset(), ) -> Tuple[Dict[str, Any], TableResult]: - """One table of the document, from one read of its records, and its summary.""" + """One table of the document, from one read of its records, and its summary. + + *keys* are the table's foreign-key columns whose children per parent are counted. + """ tally, cell_tally = tallies - binned = _binned_numbers(declared, plan) + # A foreign key's own values are each parent's children, key by key: its sum, extremes + # and quantiles describe them as a binned number's do its bins, so it withholds them too. + binned = _binned_numbers(declared, plan) | keys replaced = _ungrouped_means(declared) | binned - counted = _counted_levels(declared) + counted = _counted_levels(declared) | keys columns_doc = { column.name: _column( column, diff --git a/tests/profiler/_fake_backend.py b/tests/profiler/_fake_backend.py index 2b53997f..07e1d1a6 100644 --- a/tests/profiler/_fake_backend.py +++ b/tests/profiler/_fake_backend.py @@ -19,7 +19,8 @@ The fidelity grammar has no stand-in: it is flux-stats' own (ADR 0009), and the profile every run answers without ``--bundle`` is a fidelity block the profiler declares from the contract (``fluid_build/profiler/defaults.py``). So ``read_block``, -``measure``, ``check_edge_counts`` and ``enum_levels`` hand the work to flux-stats. +``measure``, ``check_edge_counts``, ``count_histogram`` and ``enum_levels`` hand the work +to flux-stats. """ from __future__ import annotations @@ -207,6 +208,9 @@ def measure(self, block: Any, statistic: Any, rows: Any) -> Any: def check_edge_counts(self, statistics: Any, **checks: Any) -> Any: return self._real().check_edge_counts(statistics, **checks) + def count_histogram(self, counts: Any, edges: Any, fractional_bits: int) -> Any: + return self._real().count_histogram(counts, edges, fractional_bits) + def metrics(self) -> Tuple[FakeMetric, ...]: return METRICS diff --git a/tests/profiler/test_bundle_flux_stats.py b/tests/profiler/test_bundle_flux_stats.py index 2c2c2ffd..06fe145b 100644 --- a/tests/profiler/test_bundle_flux_stats.py +++ b/tests/profiler/test_bundle_flux_stats.py @@ -174,10 +174,10 @@ def _entry(result, column: str, definition: str, group_by: Optional[str] = None) def test_the_document_answers_every_declared_property(tmp_path): result = _profile(tmp_path, _rows()) assert result.fractional_bits == 16 - assert [reason for _, reason in result.left_out] == [ + assert [reason for _, reason in result.tables[0].left_out] == [ "a correlation's moments need the exact record count, which the output checks round" ] - assert result.left_out[0][0] == "correlation of 'amount' and 'qty'" + assert result.tables[0].left_out[0][0] == "correlation of 'amount' and 'qty'" assert not [ g for e in _entries(result, "amount") @@ -290,11 +290,11 @@ def test_a_rare_support_member_withholds_the_group_and_siblings_of_a_grouped_mea assert _entry(result, "amount", "mean/1")["groups"][0]["status"] == "F" # Three siblings of the mean, and the quantile beside the histogram of its column. assert result.withheld["D"] == 4 + 1 # + the groupBy column's distinctCount - assert ("amount", "mean/1", "market", None) not in result.statistics - assert ("amount", "mean/1", None, None) in result.statistics - assert ("market", "support/1", None, None) in result.withheld_statistics - assert ("amount", "mean/1", "market", None) in result.withheld_statistics - assert ("amount", "mean/1", None, None) not in result.withheld_statistics + assert ("amount", "mean/1", "market", None) not in result.tables[0].statistics + assert ("amount", "mean/1", None, None) in result.tables[0].statistics + assert ("market", "support/1", None, None) in result.tables[0].withheld_statistics + assert ("amount", "mean/1", "market", None) in result.tables[0].withheld_statistics + assert ("amount", "mean/1", None, None) not in result.tables[0].withheld_statistics # The histogram's DE cell is a one: withheld by its cell, the group is published. (group,) = _entry(result, "market", "levelHistogram/1")["groups"] assert group["cellStatus"][0] == "C" and group["levelCounts"][0] is None @@ -533,7 +533,7 @@ def test_a_quantile_with_bins_is_written_as_quantile_2(tmp_path): (group,) = read.groups assert group.published and group.statistic.quantile_bin == 0 assert (group.statistic.bin_below, group.statistic.bin_through) == (0, 70) - assert result.statistics.count( + assert result.tables[0].statistics.count( ("amount", "quantile/2", None, "at level 0.5 over bins 0, 50, 100") ) @@ -715,7 +715,7 @@ def test_a_null_key_on_a_null_value_hides_nothing(tmp_path): def test_two_quantiles_of_one_column_are_named_apart_by_level(tmp_path): quantiles = [{**PROPERTIES[5], "level": "0.5"}, {**PROPERTIES[5], "level": "0.25"}] result = _profile(tmp_path, _rows(), quantiles) - named = result.statistics + result.withheld_statistics + named = result.tables[0].statistics + result.tables[0].withheld_statistics shapes = {each[3] for each in named if each[1] == "quantile/1"} assert shapes == {"at level 0.5", "at level 0.25"} @@ -1045,9 +1045,9 @@ def test_a_statistic_with_some_groups_withheld_is_named_with_its_counts(tmp_path assert 0 < published < len(statuses) assert set(statuses) == {"F", "C"} named = ("amount", "quantile/2", "market", "at level 0.5 over bins 0, 50, 100") - assert result.partly_withheld_statistics == ((*named, published, len(statuses)),) - assert named not in result.statistics - assert named not in result.withheld_statistics + assert result.tables[0].partly_withheld_statistics == ((*named, published, len(statuses)),) + assert named not in result.tables[0].statistics + assert named not in result.tables[0].withheld_statistics def test_the_report_names_the_statistics_withheld_in_part(tmp_path, monkeypatch, capsys): @@ -1173,7 +1173,11 @@ def test_a_histogram_group_with_no_cell_left_is_withheld_whole(tmp_path): groups = {group["group"]: group for group in entry["groups"]} assert groups["DE"] == {"group": "DE", "status": "C"} assert {key for key, group in groups.items() if group["status"] == "F"} == {"FR", "GB", "IE"} - (named,) = [one for one in result.partly_withheld_statistics if one[1] == "marginalHistogram/1"] + (named,) = [ + one + for one in result.tables[0].partly_withheld_statistics + if one[1] == "marginalHistogram/1" + ] assert named[-2:] == (3, 4) load_profile_text(result.canonical.decode("utf-8")) diff --git a/tests/profiler/test_disclosure_oracle.py b/tests/profiler/test_disclosure_oracle.py index 445bf4cf..2f6a0dd9 100644 --- a/tests/profiler/test_disclosure_oracle.py +++ b/tests/profiler/test_disclosure_oracle.py @@ -3,8 +3,8 @@ """An exhaustive audit of the count tables a profile publishes: no small count is pinned. -For tiny seeded instances (one or two columns, at most three bins, at most seven rows) -this enumerates every world, every multiset of records, runs the real profiler on each +For tiny seeded instances (one or two tables, at most three bins, at most seven rows +or six parents) this enumerates every world, every multiset of records, runs the real profiler on each (the registry, the output checks and the document, through ``profile_exposes``), and groups the worlds by the document they publish. Worlds that publish the same document are the worlds a reader cannot tell apart, the publish decision included: a withheld @@ -30,14 +30,24 @@ ones, so a number column the contract's block bins withholds them and forms no ``mean/1``; - a histogram and quantiles on the same bins, checked together, let a reader who knows - the check's choices tell worlds apart, so a column gets one or the other. + the check's choices tell worlds apart, so a column gets one or the other; +- children per parent: the check sees the parents' cells and their total, while a + reader can know far more: other tables over the same parents bound their number, and + the children's own counts bound what they hold (each child row has at most one parent, + and a parent holds between its cell's edges). So the profiler audits every withheld + cell against the strongest reader, who knows the numbers of parents and of children + exactly, and withholds more until no cell can be worked out + (:func:`test_the_strongest_reader_pins_no_small_count_either` keys worlds by those + numbers too; :func:`test_children_per_parent_under_acro_pin_no_small_count` splits a + band of children over a band of parents every way, under ACRO's rule). """ from __future__ import annotations import itertools +import json from collections import defaultdict -from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple import pytest @@ -63,6 +73,8 @@ ) for cell, base in ((2, 1), (3, 1), (3, 2), (4, 2)) ] +#: The rules that round, as every profile's counts are rounded (ACRO rounds to 5). +ROUNDING_RULES = [rule for rule in RULES if rule.rounding_base > 1] Row = Tuple[Optional[str], ...] #: A protected count: its name, and the function that counts it in a world. @@ -112,6 +124,7 @@ def _without_new_statistics(document: Mapping[str, Any]) -> str: stripped = json.loads(json.dumps(document)) stripped.pop("_schemaURL", None) + stripped.pop("relationships", None) for table in stripped["tables"].values(): for column in table["columns"].values(): kept = [s for s in column.get("statistics", []) if s["definition"] == "mean/1"] @@ -253,3 +266,238 @@ def test_quantiles_on_a_types_ladder_pin_no_small_count(monkeypatch, rule): ran, pins = _audit(monkeypatch, contract, ["v"], values, _spans("v", cells), rule) assert ran == 330 assert pins == [] + + +#: Two tables: parents, and children whose key references them. +RELATIONSHIP_CONTRACT: Mapping[str, Any] = { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "oracle", + "exposes": [ + {"exposeId": "org", "contract": {"schema": [{"name": "key", "type": "string"}]}}, + { + "exposeId": "acct", + "contract": {"schema": [{"name": "org", "type": "string"}]}, + "semantics": { + "relationships": [ + { + "name": "owner", + "to": "org", + "fromColumns": ["org"], + "toColumns": ["key"], + "cardinality": "many_to_one", + } + ] + }, + }, + ], +} + + +def _read_parents_and_children(monkeypatch: pytest.MonkeyPatch, current: Dict[str, Any]) -> None: + """Have every read return ``current["counts"]``'s parents and their children.""" + + def read_columns(source, names, *, table=None, cwd=None): + counts = current["counts"] + if source == "org": + values = [f"p{n}" for n in range(len(counts))] + return sources.SourceColumns("org", len(values), {"key": values}) + values = [f"p{n}" for n, count in enumerate(counts) for _ in range(count)] + return sources.SourceColumns("acct", len(values), {"org": values}) + + monkeypatch.setattr(sources, "read_columns", read_columns) + + +def _profile_parents(rule: disclosure.DisclosureRule, backend: Any) -> Any: + return runner.profile_exposes( + RELATIONSHIP_CONTRACT, + [runner.ExposeSource("org", "org"), runner.ExposeSource("acct", "acct")], + backend=backend, + rule=rule, + ) + + +def _relationship_pins( + monkeypatch: pytest.MonkeyPatch, rules: Sequence[disclosure.DisclosureRule] +) -> List[str]: + """Two tables: parents, and children that reference them, 0 to 2 per parent. + + A world is the parents' count and the multiset of their children's counts; which parent + has which count is not seen by any statistic of string keys. The protected counts are + the parents with none, one and two children, and every run of them. + """ + backend = load_backend() + worlds = [ + counts + for parents in range(7) + for counts in itertools.combinations_with_replacement((0, 1, 2), parents) + ] + current: Dict[str, Any] = {} + _read_parents_and_children(monkeypatch, current) + quantities = [ + (f"parents with {low} to {high - 1} children", low, high) + for low in range(3) + for high in range(low + 1, 4) + ] + pins: List[str] = [] + for rule in rules: + groups: Dict[bytes, List[Tuple[int, ...]]] = defaultdict(list) + old: Dict[str, List[Tuple[int, ...]]] = defaultdict(list) + views: Dict[Tuple[int, ...], str] = {} + for counts in worlds: + current["counts"] = counts + result = _profile_parents(rule, backend) + assert result.relationships, "the relationship is counted" + groups[result.canonical].append(counts) + views[counts] = _without_new_statistics(result.document) + old[views[counts]].append(counts) + for group in groups.values(): + if any(len(counts) > 6 - rule.rounding_base for counts in group): + continue + for name, low, high in quantities: + seen = {sum(1 for c in counts if low <= c < high) for counts in group} + if len(seen) != 1: + continue + (value,) = seen + if value >= rule.minimum_cell or (value == 0 and not rule.zeros_disclosive): + continue + before = { + sum(1 for c in counts if low <= c < high) for counts in old[views[group[0]]] + } + if len(before) > 1: + pins.append(f"{rule.name}: {name} = {value} in worlds like {group[0]}") + assert len(worlds) == 84 + return pins + + +def test_children_per_parent_pin_no_small_count(monkeypatch): + assert _relationship_pins(monkeypatch, ROUNDING_RULES) == [] + + +def test_with_exact_counts_children_per_parent_pins_no_small_count(monkeypatch): + # The audit's reader knows the children's total exactly, as exact counts publish it. + exact = [rule for rule in RULES if rule.rounding_base == 1] + assert _relationship_pins(monkeypatch, exact) == [] + + +def _splits(total: int, parts: int, largest: Optional[int] = None) -> Iterable[Tuple[int, ...]]: + """Every multiset of at most *parts* positive counts adding up to *total*.""" + largest = total if largest is None else largest + if total == 0: + yield () + return + if parts == 0: + return + for first in range(min(total, largest), 0, -1): + for rest in _splits(total - first, parts - 1, first): + yield (first, *rest) + + +def _children_pins( + monkeypatch: pytest.MonkeyPatch, + worlds: Sequence[Tuple[int, ...]], + rule: disclosure.DisclosureRule, + interior: Callable[[Tuple[int, ...]], bool] = lambda world: True, + strongest: bool = False, +) -> List[str]: + """The protected counts of children per parent that *worlds* pin under *rule*. + + A world is each parent's number of children, none an orphan. The protected counts are + the histogram's cells and every run of them; a group holding a world off the + universe's *interior* is not judged, since worlds past its edge could join it. The + *strongest* reader also knows the number of parents and of children exactly, as other + tables over the same rows can tell them: worlds are grouped by those too. + """ + backend = load_backend() + current: Dict[str, Any] = {} + _read_parents_and_children(monkeypatch, current) + lower = runner.CHILDREN_EDGES[:-1] + + def cells(world: Sequence[int]) -> List[int]: + found = [0] * len(lower) + for count in world: + found[max(index for index, edge in enumerate(lower) if count >= edge)] += 1 + return found + + groups: Dict[Tuple[Any, ...], List[Tuple[int, ...]]] = defaultdict(list) + old: Dict[str, List[Tuple[int, ...]]] = defaultdict(list) + views: Dict[Tuple[int, ...], str] = {} + for world in worlds: + current["counts"] = world + result = _profile_parents(rule, backend) + known = (len(world), sum(world)) if strongest else () + groups[(result.canonical, *known)].append(world) + views[world] = json.dumps([_without_new_statistics(result.document), *known]) + old[views[world]].append(world) + pins: List[str] = [] + for group in groups.values(): + if not all(interior(world) for world in group): + continue + for low in range(len(lower)): + for high in range(low + 1, len(lower) + 1): + seen = {sum(cells(world)[low:high]) for world in group} + (value,) = seen if len(seen) == 1 else (None,) + if value is None or value >= rule.minimum_cell: + continue + if value == 0 and not rule.zeros_disclosive: + continue + before = {sum(cells(world)[low:high]) for world in old[views[group[0]]]} + if len(before) > 1: + pins.append( + f"{rule.name}: cells {low} to {high - 1} = {value}, as in {group[0]}" + ) + return pins + + +def _split_worlds(parents: Tuple[int, int], children: Tuple[int, int]) -> List[Tuple[int, ...]]: + """Every split of every total of *children* over every count of *parents*.""" + return [ + tuple(sorted((0,) * (count - len(split)) + split)) + for count in range(parents[0], parents[1] + 1) + for total in range(children[0], children[1] + 1) + for split in _splits(total, count) + ] + + +#: A band of parents and a band of children, which ACRO's rule rounds to 25 and 10, and +#: to 30 and 15: each band whole, so every world a document fits is enumerated. +UNDER_ACRO = [((23, 27), (10, 12), 875), ((28, 32), (13, 17), 4700)] + + +@pytest.mark.parametrize("parents, children, worlds", UNDER_ACRO) +def test_children_per_parent_under_acro_pin_no_small_count(monkeypatch, parents, children, worlds): + split = _split_worlds(parents, children) + assert len(split) == worlds + assert _children_pins(monkeypatch, split, disclosure.ACRO_DEFAULT) == [] + + +def test_without_the_audit_the_childrens_counts_pin_a_zero(monkeypatch): + # The output check alone sees the parents' cells and their total, not the children's. + monkeypatch.setattr(runner, "_audited_children", lambda decision, *rest: decision) + split = _split_worlds((23, 27), (10, 12)) + assert _children_pins(monkeypatch, split, disclosure.ACRO_DEFAULT) + + +@pytest.mark.parametrize("rule", RULES, ids=lambda rule: rule.name) +def test_the_strongest_reader_pins_no_small_count_either(monkeypatch, rule): + # The number of parents and of children known exactly, every rule, exact counts too. + split = _split_worlds((0, 7), (0, 14)) + + def interior(world: Tuple[int, ...]) -> bool: + return len(world) <= 7 - rule.rounding_base and sum(world) <= 14 - rule.rounding_base + + assert _children_pins(monkeypatch, split, rule, interior, strongest=True) == [] + + +@pytest.mark.parametrize("rule", ROUNDING_RULES, ids=lambda rule: rule.name) +def test_children_per_parent_with_unbounded_children_pin_no_small_count(monkeypatch, rule): + # Up to 7 parents and 14 children split every way: under these small rules the check + # publishes cells in many worlds and not in others, so its choices and the audit's + # are part of what a reader sees. + split = _split_worlds((0, 7), (0, 14)) + assert len(split) == 1613 + + def interior(world: Tuple[int, ...]) -> bool: + return len(world) <= 7 - rule.rounding_base and sum(world) <= 14 - rule.rounding_base + + assert _children_pins(monkeypatch, split, rule, interior) == [] diff --git a/tests/profiler/test_flux_stats_backend.py b/tests/profiler/test_flux_stats_backend.py index 3633c3ad..72ea9c97 100644 --- a/tests/profiler/test_flux_stats_backend.py +++ b/tests/profiler/test_flux_stats_backend.py @@ -234,7 +234,7 @@ def test_a_boolean_column_is_left_out_as_flux_stats_reads_none(tmp_path, backend } rows = [{"attempts": n % 4, "paid": n % 2 == 0} for n in range(40)] result = profile_expose(contract, "payment", source=_slice(tmp_path, rows), backend=backend) - assert [name for name, _ in result.plan.skipped] == ["paid"] + assert [name for name, _ in result.tables[0].plan.skipped] == ["paid"] load_profile_text(result.canonical.decode("utf-8")) diff --git a/tests/profiler/test_relationships.py b/tests/profiler/test_relationships.py new file mode 100644 index 00000000..a0e27d82 --- /dev/null +++ b/tests/profiler/test_relationships.py @@ -0,0 +1,468 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""Children per parent: a histogram with the parent as unit, for each declared relationship.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from fluid_build.profiler import disclosure, runner +from fluid_build.profiler.runner import CHILDREN_EDGES, ExposeSource, profile_exposes +from tests.profiler._fake_backend import FakeBackend + +pytestmark = pytest.mark.unit + +pytest.importorskip("flux_stats.fidelity") + + +def _contract(child_key="int", relationship=None, validity=False): + org = {"exposeId": "org", "contract": {"schema": [{"name": "org_id", "type": "int"}]}} + if validity: + org["semantics"] = { + "dimensions": [ + { + "name": "from", + "type": "time", + "typeParams": {"validityParams": {"isStart": True}}, + } + ] + } + account = { + "exposeId": "account", + "contract": { + "schema": [{"name": "account_id", "type": "int"}, {"name": "org_id", "type": child_key}] + }, + "semantics": { + "relationships": [ + relationship + or { + "name": "owner", + "to": "org", + "fromColumns": ["org_id"], + "toColumns": ["org_id"], + "cardinality": "many_to_one", + } + ] + }, + } + return {"fluidVersion": "0.7.7", "kind": "DataProduct", "id": "p", "exposes": [org, account]} + + +def _write(path: Path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return str(path) + + +def _children(counts): + """Child rows giving parent n the number of children ``counts[n]``.""" + rows, next_id = [], 0 + for parent, count in enumerate(counts): + for _ in range(count): + rows.append({"account_id": str(next_id), "org_id": str(parent)}) + next_id += 1 + return rows + + +def _profile(tmp_path, contract, counts, child_rows=None, parents=None): + org = _write( + tmp_path / "org.ndjson", parents or [{"org_id": str(n)} for n in range(len(counts))] + ) + account = _write(tmp_path / "account.ndjson", child_rows or _children(counts)) + return profile_exposes( + contract, + [ExposeSource("org", org), ExposeSource("account", account)], + backend=FakeBackend(), + ) + + +def test_each_parent_counts_once_by_its_number_of_children(tmp_path): + # 20 parents with none, 30 with one, 20 with two, 10 with four, 10 with 25. + counts = [0] * 20 + [1] * 30 + [2] * 20 + [4] * 10 + [25] * 10 + result = _profile(tmp_path, _contract(), counts) + (entry,) = result.document["relationships"] + assert entry["parent"] == {"table": "org", "columns": ["org_id"]} + assert entry["child"] == {"table": "account", "columns": ["org_id"]} + (statistic,) = entry["statistics"] + assert statistic["definition"] == "marginalHistogram/1" + assert statistic["bins"] == [str(edge) for edge in CHILDREN_EDGES] + assert statistic["bins"][-1] == "9007199254740991" # 20 or more, never cut + (group,) = statistic["groups"] + assert group["k"] == 90 # the parents, rounded as the parent's rowCount is + # 0, 1, 2, 3-4, 5-9, 10-19, 20+: the empty 5-9 and 10-19 are withheld, and so is a + # neighbour, or the total would give them back. + assert group["status"] == "F" + assert group["binCounts"][:3] == [20, 30, 20] + assert group["cellStatus"][4:6] == ["C", "C"] + (counted,) = result.relationships + assert counted.relationship.name == "owner" and counted.cells == 7 + + +def test_the_foreign_keys_distinct_count_is_withheld_beside_it(tmp_path): + counts = [0] * 20 + [1] * 30 + [2] * 20 + result = _profile(tmp_path, _contract(), counts) + metrics = result.document["tables"]["account"]["columns"]["org_id"]["metrics"] + # The parents with a child: beside the parents' rowCount, it would count those with none. + assert metrics["distinctCount"] == { + "definition": "distinctCount/1", + "status": "D", + "sampleSize": metrics["count"]["sampleSize"], + "disclosure": "acro-default", + } + # Its sum, extremes and quantiles describe each parent's children key by key, and it + # forms no mean/1. + for name in ("sum", "min", "max", "quantiles"): + assert metrics[name]["status"] == "D" + assert not result.document["tables"]["account"]["columns"]["org_id"].get("statistics") + # The parent's own key keeps it. + assert ( + result.document["tables"]["org"]["columns"]["org_id"]["metrics"]["distinctCount"]["status"] + == "F" + ) + + +def test_a_null_or_unmatched_key_is_no_parents_child(tmp_path, monkeypatch): + counted = [] + audit = runner._audited_children + + def spy(decision, group, *rest): + counted.append(group.cells) + return audit(decision, group, *rest) + + monkeypatch.setattr(runner, "_audited_children", spy) + counts = [1] * 20 + [2] * 20 + rows = _children(counts) + [ + {"account_id": "1001", "org_id": None}, # no parent + {"account_id": "1002", "org_id": "999"}, # a parent the table does not hold + ] + parents = [{"org_id": str(n)} for n in range(40)] + [{"org_id": None}] * 10 + result = _profile(tmp_path, _contract(), counts, child_rows=rows, parents=parents) + ((group,),) = [entry["statistics"][0]["groups"] for entry in result.document["relationships"]] + # The ten parents whose key is null have no child, 20 have one and 20 two; the two + # children that match no parent are counted for none. + assert counted == [(10, 20, 20, 0, 0, 0, 0)] + assert group["k"] == 50 + # Beside about 60 children, the parents with one child would show that none has 20 + # or more, so the audit withholds that cell too. + assert group["binCounts"][:2] == [10, None] + + +LOWER = CHILDREN_EDGES[:-1] + + +def test_the_audit_finds_a_high_zero_the_childrens_counts_give_back(): + # 503 parents: 100 with no child, 400 with one, 3 with two, so 406 children. With only + # the cell of no child published, the other 403 parents hold 406 children: none can + # hold 10, whatever a reader moves. + cells = (100, 400, 3, 0, 0, 0, 0) + pinned = disclosure.pinned_children_cells( + LOWER, cells, [True] + [False] * 6, 406, disclosure.ACRO_DEFAULT + ) + assert {5, 6} <= pinned + + +def test_a_cell_with_room_to_move_is_not_pinned(): + # 60 parents: 20 with no child, 20 with one, 20 with 25. A reader who knows the 520 + # children exactly can still move children between the parents of the top cell. + cells = (20, 20, 0, 0, 0, 0, 20) + published = [True, True, False, False, False, False, False] + pinned = disclosure.pinned_children_cells(LOWER, cells, published, 520, disclosure.ACRO_DEFAULT) + assert pinned == set() + + +def test_the_childrens_total_bounds_the_cells_from_below_too(): + # One child among 30 parents: a parent must hold it. With every other cell published, + # the cell of no child cannot gain the one parent that holds it. + cells = (29, 1, 0, 0, 0, 0, 0) + published = [False] + [True] * 6 + pinned = disclosure.pinned_children_cells(LOWER, cells, published, 1, disclosure.ACRO_DEFAULT) + assert pinned == {0} + + +def test_the_check_never_leaves_a_small_total_withheld(): + # The one withheld cell holds 10 parents: the check would have withheld more had it + # held fewer, so a reader knows it cannot have lost a parent to another cell. + cells = (10, 0, 0, 10, 0, 0, 0) + published = [True, True, True, False, True, True, True] + pinned = disclosure.pinned_children_cells(LOWER, cells, published, 35, disclosure.ACRO_DEFAULT) + assert pinned == {3} + + +def test_a_table_the_readers_model_does_not_hold_claims_nothing_safe(): + # 12 parents with a null key cannot all sit in a cell of 11 parents: every withheld + # cell is returned, so the audit withholds the histogram whole. + cells = (11, 12, 0, 0, 0, 0, 0) + pinned = disclosure.pinned_children_cells( + LOWER, cells, [True] + [False] * 6, 12, disclosure.ACRO_DEFAULT, childless=12 + ) + assert pinned == set(range(1, 7)) + + +@pytest.mark.parametrize( + "counts", + [[0] * 100 + [1] * 400 + [2] * 3, [0] * 12 + [1] * 11], + ids=["503 parents", "23 parents"], +) +def test_no_withheld_cell_follows_from_the_childrens_counts(tmp_path, counts): + result = _profile(tmp_path, _contract(), counts) + ((group,),) = [entry["statistics"][0]["groups"] for entry in result.document["relationships"]] + # The check published the cell of no child; beside the children's counts it pinned a + # high bin's zero, so the histogram is withheld whole. + assert group["status"] == "C" + + +def test_an_enum_of_the_parents_does_not_give_a_withheld_cell_back(tmp_path): + # The parents' level histogram bounds their number (10 + 13 + 18, and at least 10 + # withheld), tighter than their row count: beside the 41 children, no parent can hold + # 20. The strongest reader knows that number exactly, so the histogram is withheld. + counts = [0] * 27 + [1] * 14 + [2] * 6 + [3] * 5 + tiers = ["a"] * 10 + ["b"] * 13 + ["c"] * 18 + ["d"] * 2 + ["e"] * 9 + contract = _contract() + contract["exposes"][0]["contract"]["schema"].append( + {"name": "tier", "type": "enum('a', 'b', 'c', 'd', 'e')"} + ) + parents = [{"org_id": str(n), "tier": tier} for n, tier in enumerate(tiers)] + result = _profile(tmp_path, contract, counts, parents=parents) + tier = result.document["tables"]["org"]["columns"]["tier"]["statistics"][0]["groups"][0] + assert tier["levelCounts"][:3] == [10, 15, 20] + ((group,),) = [entry["statistics"][0]["groups"] for entry in result.document["relationships"]] + assert group["status"] == "C" + + +def test_two_relationships_into_one_parent_do_not_give_each_other_away(tmp_path): + # Each relationship's cells bound the parents' number for the other one. + first = [0] * 27 + [1] * 14 + [2] * 6 + [3] * 5 + second = [0] * 10 + [1] * 13 + [2] * 18 + [3] * 2 + [5] * 9 + relationship = { + "name": "owner", + "to": "org", + "fromColumns": ["org_id"], + "toColumns": ["org_id"], + "cardinality": "many_to_one", + } + contract = _contract() + contract["exposes"].append( + { + "exposeId": "invoice", + "contract": {"schema": [{"name": "org_id", "type": "int"}]}, + "semantics": {"relationships": [dict(relationship, name="billed")]}, + } + ) + org = _write(tmp_path / "org.ndjson", [{"org_id": str(n)} for n in range(52)]) + account = _write(tmp_path / "account.ndjson", _children(first)) + invoice = _write( + tmp_path / "invoice.ndjson", + [{"org_id": str(n)} for n, count in enumerate(second) for _ in range(count)], + ) + result = profile_exposes( + contract, + [ + ExposeSource("org", org), + ExposeSource("account", account), + ExposeSource("invoice", invoice), + ], + backend=FakeBackend(), + ) + groups = { + entry["child"]["table"]: entry["statistics"][0]["groups"][0] + for entry in result.document["relationships"] + } + assert groups["account"]["status"] == "C" + for name, counts in (("account", first), ("invoice", second)): + cells = [0] * len(LOWER) + for count in counts: + cells[max(i for i, edge in enumerate(LOWER) if count >= edge)] += 1 + group = groups[name] + if group["status"] == "F": # what it publishes leaves no withheld cell pinned + published = [status == "F" for status in group["cellStatus"]] + assert not disclosure.pinned_children_cells( + LOWER, cells, published, sum(counts), disclosure.ACRO_DEFAULT + ) + + +def test_a_relationship_whose_parent_is_not_profiled_is_named_and_left_out(tmp_path): + account = _write(tmp_path / "account.ndjson", _children([1] * 20)) + result = profile_exposes(_contract(), [ExposeSource("account", account)], backend=FakeBackend()) + assert "relationships" not in result.document + assert result.relationships_left_out == ( + ( + "account", + "owner", + "its target is not profiled in this call: profile both exposes together", + ), + ) + # No parent is counted, so the key's distinct count is not withheld for it. + metrics = result.document["tables"]["account"]["columns"]["org_id"]["metrics"] + assert metrics["distinctCount"]["status"] != "D" + + +def test_keys_read_in_different_domains_are_named_and_left_out(tmp_path): + result = _profile(tmp_path, _contract(child_key="string"), [1] * 20) + assert "relationships" not in result.document + ((child, name, reason),) = result.relationships_left_out + assert (child, name) == ("account", "owner") and "different domains" in reason + + +@pytest.mark.parametrize("columns", [5, "org_id", {"name": "org_id"}]) +def test_key_columns_that_are_not_lists_are_named_and_left_out(tmp_path, columns): + relationship = {"name": "owner", "to": "org", "fromColumns": columns, "toColumns": columns} + result = _profile(tmp_path, _contract(relationship=relationship), [1] * 20) + assert "relationships" not in result.document + ((_, name, reason),) = result.relationships_left_out + assert name == "owner" and "not two lists" in reason + + +def test_the_document_does_not_depend_on_the_order_the_exposes_are_named_in(tmp_path): + def expose(name, parent=None): + schema = [{"name": f"{name}_id", "type": "int"}] + entry = {"exposeId": name, "contract": {"schema": schema}} + if parent is not None: + key = f"{parent}_id" + schema.append({"name": key, "type": "int"}) + entry["semantics"] = { + "relationships": [ + {"name": f"of_{parent}", "to": parent, "fromColumns": [key], "toColumns": [key]} + ] + } + return entry + + contract = { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "p", + "exposes": [expose("org"), expose("account", "org"), expose("invoice", "account")], + } + sources = { + "org": _write(tmp_path / "org.ndjson", [{"org_id": str(n)} for n in range(20)]), + "account": _write( + tmp_path / "account.ndjson", + [{"account_id": str(n), "org_id": str(n % 20)} for n in range(40)], + ), + "invoice": _write( + tmp_path / "invoice.ndjson", + [{"invoice_id": str(n), "account_id": str(n % 40)} for n in range(80)], + ), + } + digests = set() + for order in (["org", "account", "invoice"], ["invoice", "account", "org"]): + result = profile_exposes( + contract, [ExposeSource(each, sources[each]) for each in order], backend=FakeBackend() + ) + children = [entry["child"]["table"] for entry in result.document["relationships"]] + assert children == ["account", "invoice"] # the contract's order + digests.add(result.digest) + assert len(digests) == 1 + + +def _edge_statistics(result, table, column): + statistics = result.document["tables"][table]["columns"][column].get("statistics", []) + return [each["definition"] for each in statistics if each["definition"] != "mean/1"] + + +def test_an_enum_key_takes_no_level_histogram_beside_children_per_parent(tmp_path): + # An enum key's level histogram counts each key's rows, and the child key's rows are + # each parent's children: the keys take none, and the relationship is counted. + keys = [f"L{n:02d}" for n in range(30)] + enum = "enum(" + ", ".join(f"'{key}'" for key in keys) + ")" + contract = _contract(child_key=enum) + contract["exposes"][0]["contract"]["schema"][0]["type"] = enum + counts = [0] * 10 + [12] * 20 + parents = [{"org_id": key} for key in keys] + rows = [ + {"account_id": str(n), "org_id": key} + for n, (key, count) in enumerate(zip(keys, counts, strict=True)) + for _ in range(count) + ] + result = _profile(tmp_path, contract, counts, child_rows=rows, parents=parents) + assert result.relationships and not result.relationships_left_out + assert _edge_statistics(result, "account", "org_id") == [] + assert _edge_statistics(result, "org", "org_id") == [] + uncovered = {name: why for table in result.tables for name, why in table.uncovered} + assert "counted relationship" in uncovered["org_id"] + + +@pytest.mark.parametrize( + "rules", [[], [{"type": "range", "constraint": "between 0 and 29"}]], ids=["ladder", "range"] +) +def test_a_number_key_takes_no_histogram_or_quantile(tmp_path, rules): + # A declared range would give each key a bin of its own, and the ladder's bins near 0 + # are one unit wide: either would count each parent's children beside the cells. + contract = _contract() + for expose in contract["exposes"]: + for column in expose["contract"]["schema"]: + if column["name"] == "org_id": + column["validationRules"] = rules + result = _profile(tmp_path, contract, [0] * 10 + [12] * 20) + assert result.relationships + assert _edge_statistics(result, "account", "org_id") == [] + assert _edge_statistics(result, "org", "org_id") == [] + # A column that is no key keeps its statistics. + assert _edge_statistics(result, "account", "account_id") != [] + + +def test_a_target_with_a_validity_window_is_left_out(tmp_path): + result = _profile(tmp_path, _contract(validity=True), [1] * 20) + assert "relationships" not in result.document + assert "validity window" in result.relationships_left_out[0][2] + + +def test_a_self_join_counts_each_row_as_a_parent(tmp_path): + contract = { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "p", + "exposes": [ + { + "exposeId": "employee", + "contract": { + "schema": [{"name": "id", "type": "int"}, {"name": "manager", "type": "int"}] + }, + "semantics": { + "relationships": [ + { + "name": "reports_to", + "to": "employee", + "fromColumns": ["manager"], + "toColumns": ["id"], + "cardinality": "many_to_one", + } + ] + }, + } + ], + } + # 10 managers with 3 reports each; the 30 reports and the managers manage nobody else. + rows = [{"id": str(m), "manager": None} for m in range(10)] + [ + {"id": str(100 + n), "manager": str(n % 10)} for n in range(30) + ] + source = _write(tmp_path / "employee.ndjson", rows) + result = profile_exposes(contract, [ExposeSource("employee", source)], backend=FakeBackend()) + ((group,),) = [entry["statistics"][0]["groups"] for entry in result.document["relationships"]] + assert group["k"] == 40 + assert group["binCounts"][0] == 30 # the reports have no one under them + assert result.relationships[0].relationship.parent == "employee" + + +def test_the_document_reads_back_with_its_relationship(tmp_path): + from flux_stats.profile import load_profile_text + + counts = [0] * 20 + [1] * 30 + [3] * 20 + result = _profile(tmp_path, _contract(), counts) + profile, _ = load_profile_text(result.canonical.decode("utf-8")) + (relationship,) = profile.relationships + assert (relationship.parent_table, relationship.child_table) == ("org", "account") + assert relationship.statistics[0].identifier == "marginalHistogram/1" + + +def test_the_summary_names_each_relationship(tmp_path, capsys): + from fluid_build.cli.profile_cmd import _report + + result = _profile(tmp_path, _contract(), [0] * 20 + [1] * 30 + [2] * 20) + lines = [] + _report(lambda text, **_: lines.append(text), result, tmp_path / "out.json") + text = " ".join(lines) + assert "Children per parent of exposes[org] in exposes[account] (owner):" in text + assert runner.RelationshipResult is type(result.relationships[0]) diff --git a/tests/profiler/test_runner.py b/tests/profiler/test_runner.py index 5eb6d3ac..1d4c770f 100644 --- a/tests/profiler/test_runner.py +++ b/tests/profiler/test_runner.py @@ -228,7 +228,7 @@ def test_a_number_column_carries_mean_1_on_the_documents_grid(tmp_path): "placed_at": ["quantile/2"], "note": [], } - assert ("amount", "mean/1", None, None) in result.statistics + assert ("amount", "mean/1", None, None) in result.tables[0].statistics assert result.fractional_bits == 16 @@ -365,7 +365,7 @@ def test_a_mean_is_withheld_where_the_sum_is(tmp_path, values, status): statistic, group = _mean(result) assert group == {"group": "", "status": status} assert statistic["sampleSize"] == {"min": 38, "max": 42} - assert not [each for each in result.statistics if each[1] == "mean/1"] + assert not [each for each in result.tables[0].statistics if each[1] == "mean/1"] assert "1234567" not in result.canonical.decode("utf-8") # no edge of the ladder either @@ -524,8 +524,6 @@ def test_several_exposes_go_into_one_document_each_from_its_own_source(tmp_path) assert tables["org"]["rowCount"] == 10 and tables["account"]["rowCount"] == 20 assert [table.plan.expose_id for table in result.tables] == ["account", "org"] assert result.digest == digest_of(result.canonical) - with pytest.raises(ValueError, match="one result per expose"): - result.plan # noqa: B018 - a profile of two exposes has no one plan def test_an_expose_named_twice_is_refused_before_any_read(tmp_path): @@ -546,7 +544,7 @@ def test_one_expose_profiles_as_it_always_did(tmp_path): ORDERS, [runner.ExposeSource("orders", _slice(tmp_path, rows))], backend=FakeBackend() ) assert one.canonical == many.canonical - assert one.plan == many.plan + assert one.tables == many.tables # -- a column the contract types date -----------------------------------------------------