diff --git a/AGENTS.md b/AGENTS.md index bec70145..eae7965b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,7 +164,7 @@ forge-cli/ | Bundle builder | `fluid_build/forge/core/bundle.py` — deterministic tgz + MANIFEST + fragment extraction + `$source` sentinel | | Build runners | `fluid_build/build_runners/{dbt,python}/` — extracted from legacy `cli/execute.py` in Phase 1 | | Masking at landing | `fluid_build/build_runners/_masking.py` — the DuckDB runner applies `exposes[].policy.privacy.masking[]` inside the COPY (file, S3, BigQuery staging, DLQ); secrets only from env (`FLUID_PII_HASH_SECRET`, `FLUID_PII_TOKENIZATION_KEY`, `FLUID_PII_ENCRYPTION_SECRET_KEY`, or `params.saltEnv`/`keyEnv`), held in UDF closures, never in SQL; an unappliable rule refuses the build, and so does a DuckDB without numpy (its `create_function` needs it; `[local]` names numpy, and the refusal names the extra). `cli/_verify_masking.py` is stage 9's shape check (local DuckDB, Athena in the count query) | -| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads one expose's records where they live (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC 1-0-0 document (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). With `--bundle DIR` it also answers the fidelity block of DIR's one Simulation: `bundle.py` reads `*.flux.yml` as the engine does (flux-stats' `flux_stats.exact_yaml` and `flux_stats.canonical`, through the seam, one `ReadBudget` per bundle), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a quantile with bins is `quantile/2` (the profile then names format 1-1-0), the histograms and quantiles of one column go to flux-stats' `check_edge_counts` in one call (FLUX ADR 0012; `runner._edge_decisions`), a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | +| FLUX profiler (`fluid profile`, `[profile]` extra) | `fluid_build/profiler/`: reads each expose's records where they live, every expose of one call into one document (`runner.profile_exposes`; the CLI pairs `--source =` and `--table =` with repeated `--expose`) (`sources.py`: an NDJSON slice as flux-engine reads one; CSV/Parquet and postgres/mysql/sqlite through the DuckDB sandbox and `cli/discover/_jdbc_introspect.attach_jdbc_source`), applies ACRO's output checks (`disclosure.py`) and writes a PROFILE-SPEC document, format 1-0-0 or, once it holds a `quantile/2`, 1-1-0 (`document.py`, RFC 8785, digest `sha256:`): the OpenLineage column metrics, plus an ungrouped `mean/1` per number column on the document's grid (`--fractional-bits`, default 16), published only where the column's `count` and `sum` are (`disclosure.check_mean`). Without `--bundle` it declares, from the contract alone, a fidelity block of the statistics `fluxsim init` reads (`defaults.py`: a `levelHistogram/1` over every level of an `enum(...)` column, a `marginalHistogram/1` over a `validationRules` range of two ends inside the type (each value checked against the range exactly, `DeclaredRange`), else `quantile/2` on a ladder fixed by the type for numbers and dates, grouped level histograms per categorical `semantics.dimensions` level, at most `defaults.MOST_GROUPED_CELLS` cells per column; `DeclaredBlock.every_declared_group` lists every declared level as a group) and answers it through the same path; `tests/profiler/test_disclosure_oracle.py` enumerates every tiny world to show no new statistic pins a protected count, which is why a column never gets a histogram and quantiles together and why `runner._counted_levels` / `_binned_numbers` withhold `distinctCount`, and a declared range's `sum`, `min`, `max` and `quantiles`. With `--bundle DIR` it answers the fidelity block of DIR's one Simulation instead: `bundle.py` reads `*.flux.yml` as the engine does (flux-stats' `flux_stats.exact_yaml` and `flux_stats.canonical`, through the seam, one `ReadBudget` per bundle), flux-stats' `flux_stats.fidelity` parses the block (no parser here), and each declared property's statistic is formed per group on the block's grid, every group and cell output-checked; a quantile with bins is `quantile/2` (the profile then names format 1-1-0), the histograms and quantiles of one column go to flux-stats' `check_edge_counts` in one call (FLUX ADR 0012; `runner._edge_decisions`), a withheld group of a grouped mean or support withholds its siblings under `D`, and a correlation is not answered. Every value is read, and every statistic formed, by FLUX's registry `flux-stats` through the one seam `_stats.py`, which imports it only when a profile runs; never import `flux_engine` here (`tests/profiler/test_boundary.py`) | | CI template generator | `fluid_build/forge/core/pipeline_templates.py` — 7-system generator; Jenkins ships the full 11-stage parameterized template with `--install-mode {pypi,dev-source}` | | Forge (AI creation) | `fluid_build/forge/` (templates, generators, extensions) | | Log redaction (global) | `fluid_build/observability/secret_redactor.py` — `SecretRedactingFilter` wired into Python logging | diff --git a/CHANGELOG.md b/CHANGELOG.md index 45ad62b3..df153cc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,89 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`fluid profile` publishes the statistics `fluxsim init` reads, without a bundle.** With no + `--bundle`, the profiler declares a fidelity block from the contract alone, never from the + records, and answers it through the same path as a Simulation's block, so flux-stats parses + and checks every statistic (`check_edge_counts`, ADR 0012, and the ACRO rules): + - every `enum(...)` column gets `levelHistogram/1` over every declared level; a level that + no record holds is listed, its cell withheld as `C`, so `fluxsim init` no longer drops a + histogram that lacked it; + - a number whose `validationRules` declare both ends of a range, each inside its type, gets + `marginalHistogram/1` over about ten bins of one width, exact at both ends (an integer range + ends half a unit past its top, a whole unit at `--fractional-bits 0`, so each value has a + bin of its own); + - any other number of a bounded type (a range of one side, or with a side at the type's own + bound, included), and every date and timestamp, gets `quantile/2` at 0.25, 0.5 and 0.75 on + a ladder fixed by the type: 0, ±1, ±2, ±5, ±10 ... within the type's bounds for a number, + one bin per year from 1900 to 2100 for a date or a timestamp; + - each categorical `semantics.dimensions` entry on an `enum` column groups the other `enum` + columns' level histograms by its levels, every declared level a group, one that no record + holds included; one column's groupings hold at most 10,000 cells together, and a grouping + past that is left out and named, since checking them costs about the cube of the levels. + A number with neither a range of two ends nor a bounded type (a `decimal` without `(p,s)`) + gets no such statistic, and the summary says so. A record outside what the contract declares + (a level its type does not list, a value outside its type or its declared range, a strict + bound included) now refuses the profile with `profile_record_outside_declared_domain`, naming + the column and the row, never the value. A profile that holds a `quantile/2` names format + `1-1-0`, which flux-stats reads from 0.7, so a profile without `--bundle` of a contract with a + bounded number, a date or a timestamp is now a `1-1-0` document, with new bytes and digest. +- **Every new count table is audited for differencing, world by world.** A test profiles every + world of up to seven records for each kind of table under scaled-down ACRO rules, groups the + worlds by the document they publish, and fails on a count of one record or of none that the + new statistics pin. Run on larger universes under the ACRO defaults (up to 45 records, 46,376 + worlds for a grouping), it found three channels, which the profiler now closes: + - a column's `distinctCount` says how many withheld cells hold no record, so a column a + level histogram counts, or that a grouped statistic groups by, withholds it under `D`, + with or without `--bundle`; + - a number's exact `sum`, and its `min`, `max`, `quantiles` and `distinctCount`, give a + withheld bin of a declared range back, so a number with a default histogram withholds them + under `D` and has no `mean/1`; + - a histogram and quantiles on the same bins, checked together, let a reader who knows the + check's choices tell worlds apart, so a column gets one or the other. + With these, the audit finds no such pin under the ACRO defaults. +- **`fluid profile` profiles a whole data product in one call.** `--expose` repeats, and each + expose takes its records as `--source =` (and a database's table as + `--table =
`); with no `--expose`, every expose a source names is profiled, + in the contract's order. The result is one profile document whose `tables` hold every expose, + written to `.profile.json` unless `--out` names it, and read back whole before it + is written. One `--expose` with one bare `--source` works as before. A source that pairs with + no expose, an expose without a source, a source no `--expose` names and a second source for + one expose are refused before any record is read, and a database source's password is masked + in every refusal. With `--bundle`, each expose answers its own block, and the blocks must + declare one grid. +- **FLUID SQL/1 moves a date by whole days, months and years.** `ADD_DAYS(d, n)`, + `ADD_MONTHS(d, n)` and `ADD_YEARS(d, n)` take a date and a count of scale 0 and give a date, + with flux-expr/3's meaning (`RFC-data-rules.md` §2.6, amended): a month or year move keeps the + day of the month or clamps it to the last day of a shorter month (31 January plus one month is + 28 or 29 February, and 28 February plus one month is 28 March), a NULL argument gives NULL, + and a result outside 0001-01-01 to 9999-12-31 is NULL, never an error, however large the + count. Every dialect renders them with a guarded count, so no engine adds an out-of-range + count (§2.8), and Snowflake gets `DATEADD`, never its own `ADD_MONTHS`, which keeps month + ends. The conformance vectors are version 1.1.0, with 51 new vectors at the month ends, at + the edges of the range and for each refusal. + +### Fixed + +- **`fluid profile` reads a CSV date column.** A column the contract types `date` reads a + `YYYY-MM-DD` text as that day's midnight in UTC, as a Parquet date always read, from any + source. A CSV date column used to be refused (FLX-5022). The text is read strictly: anything + but a calendar day written `YYYY-MM-DD` goes to the reader as before, which refuses it, and + DuckDB's own date reader is not used, because it reads `infinity` and `epoch` as dates and + drops a time of day. +- **`fluid profile` no longer reads one expose's records as another's.** argparse kept the last + of two `--expose` flags, so `--expose org --expose account --source org.csv` profiled + `org.csv` as the `account` expose. Several `--expose` flags now take one + `--source =` each, and are refused before any record is read without + them (`profile_source_unpaired`); an expose named twice is `profile_expose_repeated`. +- **A grouped statistic's `sampleSize` no longer gives back the records a null `groupBy` key + leaves out.** It banded the records its groups hold. Beside the column's own `count`, the + `groupBy` column's and the row count, that band pinned how many records hold a value and a + null key, a small count, under rules that round and under exact counts alike (the disclosure + audit found it once both columns held nulls; `--bundle` profiles had it too). It now bands the + column's non-null records, as the column's ungrouped statistics do. + ## [0.35.0] — 2026-11-19 ### Added diff --git a/RFC-data-rules.md b/RFC-data-rules.md index 2f025316..1f4257c9 100644 --- a/RFC-data-rules.md +++ b/RFC-data-rules.md @@ -14,7 +14,7 @@ On 2026-11-16 the founder decided where these rules live: - **Decision 12.** Rules about what valid data looks like belong in the FLUID contract. There, dbt, ODCS and catalog consumers reuse them, and FLUX reads them instead of restating them. Distributions, behavior and run settings stay in FLUX documents. Measured statistics stay in separate profile documents (`RFC-data-profile-port.md`). - **Decision 13.** Invariants are written in a SQL expression subset that this RFC defines. flux-expr/2 implements exactly that subset. -- **Decision 14.** Date arithmetic is whole-unit differences (`YEARS_BETWEEN`, `MONTHS_BETWEEN`, `DAYS_BETWEEN`) with an explicit as-of date. The results are exact integers. +- **Decision 14.** Date arithmetic is whole-unit differences (`YEARS_BETWEEN`, `MONTHS_BETWEEN`, `DAYS_BETWEEN`) with an explicit as-of date. The results are exact integers. *Amended 2026-11-19 by the maintainers:* a date also moves by whole units (`ADD_DAYS`, `ADD_MONTHS`, `ADD_YEARS`, §2.6), with the meaning flux-expr/3 gives them (flux-engine ADR 0013), so a rule such as "a term ends 12 months after it starts" is written in the contract. This reverses the first version's choice to leave date addition out. The 0.7.7 schema cannot say any of this today: @@ -203,7 +203,7 @@ An invariant's share is a threshold on the number of covered rows that violate i - The numbers are exact decimals: a consumer reads `0.5` as one half, from its text, never as the nearest binary double. They have at most 15 significant digits, so the text survives a round trip through a double (FLUX decision Q4). - "Holds on about 98 % of real rows" is `mustBeBetween: [1.5, 2.5], unit: percent`. -The band is a rule; the measured share is a statistic. `fluid profile` (decision 11) measures each declared invariant on the real data and writes the result to a profile document. Because zero violations are disclosive, it reports "at least X %" (decision Q9). The contract never holds a measured number. FLUX will choose where in the band a run generates (a FLUX setting, which may come from a profile), inject the violations, and check the run against the band. FLUX will check a run against the band itself, as every consumer does (§2.9, change 7), and generate from a `percent` band by choosing a target strictly inside it. Three bands cannot be met in expectation: a zero-width `percent` band above 0 (`mustBe: 2`), a `rows` band other than zero violations (a count does not scale to a run of another size), and a dependency band that allows no holding share above one half, so that every value it allows is at least 50 % of violating rows (EXPR-SPEC-2 §9 generates a dependency only above that). FLUX will check such a band, generate no violation for it, and, under strictness, refuse one that zero violations would fail. +The band is a rule; the measured share is a statistic. `fluid profile` (decision 11) is planned to measure each declared invariant on the real data and write the result to a profile document; it does not measure invariants yet, and today's profile holds column statistics only. Because zero violations are disclosive, it reports "at least X %" (decision Q9). The contract never holds a measured number. FLUX will choose where in the band a run generates (a FLUX setting, which may come from a profile), inject the violations, and check the run against the band. FLUX will check a run against the band itself, as every consumer does (§2.9, change 7), and generate from a `percent` band by choosing a target strictly inside it. Three bands cannot be met in expectation: a zero-width `percent` band above 0 (`mustBe: 2`), a `rows` band other than zero violations (a count does not scale to a run of another size), and a dependency band that allows no holding share above one half, so that every value it allows is at least 50 % of violating rows (EXPR-SPEC-2 §9 generates a dependency only above that). FLUX will check such a band, generate no violation for it, and, under strictness, refuse one that zero violations would fail. ### 1.5 What a check counts @@ -237,7 +237,7 @@ FLUX will refuse, with an FLX code, an invariant it cannot generate as declared: ## 2. FLUID SQL/1, the expression subset -FLUID SQL/1 is the language of `check`, `expr`, `when` and `aggregate.filter`. It is a closed subset of SQL: a text the grammar does not derive is a validation error, never an extension point. Growth goes to FLUID SQL/2 in a later schema version. There is no per-expression language key, because the contract's `fluidVersion` names the version: 0.7.7 carries FLUID SQL/1. +FLUID SQL/1 is the language of `check`, `expr`, `when` and `aggregate.filter`. It is a closed subset of SQL: a text the grammar does not derive is a validation error, never an extension point. Growth goes to FLUID SQL/2 in a later schema version. One addition was made in place while 0.7.7 is a preview: the three date moves of §2.6 (2026-11-19, decision 14 as amended). It is additive, and it fails closed: a reader that predates it refuses a text that uses one as an unknown function, so it never computes another value. Conformance vectors 1.1.0 hold them, and 1.0.0 did not. There is no per-expression language key, because the contract's `fluidVersion` names the version: 0.7.7 carries FLUID SQL/1. ### 2.1 Grammar @@ -258,8 +258,10 @@ factor = [ "-" ] , primary ; primary = literal | column_ref | call | case_expr | "(" , expr , ")" ; call = "ROUND" , "(" , value , "," , integer , [ "," , round_mode ] , ")" | "COALESCE" , "(" , expr , "," , expr , { "," , expr } , ")" - | date_diff , "(" , value , "," , value , ")" ; + | date_diff , "(" , value , "," , value , ")" + | date_add , "(" , value , "," , value , ")" ; date_diff = "DAYS_BETWEEN" | "MONTHS_BETWEEN" | "YEARS_BETWEEN" ; +date_add = "ADD_DAYS" | "ADD_MONTHS" | "ADD_YEARS" ; round_mode = "'HALF_AWAY_FROM_ZERO'" | "'HALF_TO_EVEN'" ; case_expr = "CASE" , "WHEN" , expr , "THEN" , expr , { "WHEN" , expr , "THEN" , expr } , [ "ELSE" , expr ] , "END" ; @@ -309,9 +311,9 @@ Type rules: - `=`, `<>`, `!=`, `IS [NOT] DISTINCT FROM` and `IN` take operands of one type. Numbers of different scales compare by value. - `<`, `<=`, `>`, `>=` and `BETWEEN` take numbers, strings, dates or timestamps, one type throughout. Strings compare by the code points of the text as stored; only the contract's own texts (names, literals, placeholder values and prefixes) are NFC-normalized. A date never meets a timestamp, and a timestamp never meets a timestamp with time zone, because FLUID SQL/1 has no `CAST` and the two readings of a zoneless value differ by engine. -- `+`, `-`, `*`, `/` and unary `-` take numbers. There is no date or string arithmetic beyond the date differences. +- `+`, `-`, `*`, `/` and unary `-` take numbers. There is no date or string arithmetic beyond the date differences and the date moves. A date move takes a date and a count, a number of static scale 0 (§2.4): a count such as `1.5`, a `decimal(12,2)` column, a float column or a quotient is refused, and `ROUND(x, 0)` is accepted. - The value branches of `CASE` and `COALESCE` share one type. The `NULL` literal may stand for any of them, but not for all of them. -- More generally, the `NULL` literal takes its type from a typed operand beside it: an arithmetic operand, a date difference's argument, an `IS [NOT] DISTINCT FROM` operand, a `CASE` or `COALESCE` branch. Anywhere else (an `IN` item, an operand of a comparison or `BETWEEN`, `IS [NOT] NULL`, unary minus, `ROUND`'s argument, a predicate position, a whole value) it is refused, and so is a construct whose operands are all `NULL` literals. +- More generally, the `NULL` literal takes its type from a typed operand beside it: an arithmetic operand, a date difference's argument, an `IS [NOT] DISTINCT FROM` operand, a `CASE` or `COALESCE` branch. Anywhere else (an `IN` item, an operand of a comparison or `BETWEEN`, `IS [NOT] NULL`, unary minus, `ROUND`'s argument, either argument of a date move, a predicate position, a whole value) it is refused, and so is a construct whose operands are all `NULL` literals. - A predicate position takes a bool: `check`, `when`, `aggregate.filter`, `CASE WHEN` and the operands of `AND`, `OR` and `NOT`. - A string literal compared with a column whose `validationRules` declare an `enum` must be one of its values. This is flux-expr/1's closed-set rule, and it turns a typo such as `kind = 'base_fe'` into a validation error. @@ -354,19 +356,24 @@ Why two-valued: a row whose predicate is not TRUE does not satisfy it. Deequ's ` | `CASE WHEN c THEN a ... [ELSE b] END` | as in SQL; without `ELSE`, NULL | | `DAYS_BETWEEN(a, b)` | the whole days from date a to date b | | `MONTHS_BETWEEN(a, b)`, `YEARS_BETWEEN(a, b)` | the whole months or years from date a to date b | +| `ADD_DAYS(d, n)` | date d moved n days forward, or back when n is negative | +| `ADD_MONTHS(d, n)`, `ADD_YEARS(d, n)` | date d moved n months or years, the day of the month kept, or clamped to the last day of a shorter month | | `DATE 'YYYY-MM-DD'` | a date literal: a valid calendar date from 0001-01-01 to 9999-12-31 | The date differences use EXPR-SPEC-2 §6's definition (decision 14). For b on or after a, the result is the largest k such that a moved forward by k units is not after b. Moving forward by months or years keeps the day of the month, or lands on the month's last day when that month is shorter: 31 January 2026 plus one month is 28 February 2026, and 29 February 2000 plus 26 years is 28 February 2026. For b before a, the result is minus the result from b to a. Both arguments are dates, and a NULL argument gives NULL. Age is `YEARS_BETWEEN(birth_date, snapshot_date)`. -Two warnings for SQL readers: +The date moves are flux-expr/3's (flux-engine ADR 0013, EXPR-SPEC-3 §4), so FLUX reads them with the same meaning. `n` is a whole number. A move by months keeps the day of the month, or lands on the last day of the target month when that month is shorter: 31 January 2001 plus one month is 28 February 2001, and 31 January 2004 plus one month is 29 February 2004. The month end is not kept: 28 February 2001 plus one month is 28 March 2001. `ADD_YEARS(d, n)` is `ADD_MONTHS(d, 12 * n)`, so 29 February 2000 minus one year is 28 February 1999, and plus four years is 29 February 2004. A NULL argument gives NULL, and a result outside 0001-01-01 to 9999-12-31 is NULL, never an error, however large `n` is. The moves invert the differences: `MONTHS_BETWEEN(d, ADD_MONTHS(d, n))` is `n` whenever the result is in range, and so for days and years. + +Three warnings for SQL readers: - These are not `DATEDIFF`, `DATE_DIFF` or DuckDB's `date_diff`, which count the unit boundaries crossed. `DATEDIFF(month, '2026-01-31', '2026-02-01')` is 1, while `MONTHS_BETWEEN(DATE '2026-01-31', DATE '2026-02-01')` is 0. They equal DuckDB's `date_sub`. - Oracle, Snowflake and Spark have a `MONTHS_BETWEEN(d1, d2)` of their own. It returns a fraction, d1 minus d2, so its arguments run the other way. No consumer passes FLUID's `MONTHS_BETWEEN` to an engine by name (§2.8). +- Oracle, Db2 and Snowflake have an `ADD_MONTHS(d, n)` of their own that keeps the month end: there, 28 February 2001 plus one month is 31 March 2001. No consumer passes FLUID's `ADD_MONTHS` to an engine by name either (§2.8). ### 2.7 Not in FLUID SQL/1 - String functions and `LIKE`, regular expressions, `CAST`, timestamp literals and functions over timestamps; -- date arithmetic other than the differences, `ABS`, `LEAST`, `GREATEST`, `NULLIF`, parameters, user functions and the clock; +- date arithmetic other than the differences and the moves by whole days, months and years (no interval type, `DATE_TRUNC` or `LAST_DAY`), `ABS`, `LEAST`, `GREATEST`, `NULLIF`, parameters, user functions and the clock; - aggregates and windows inside expressions, subqueries, paths over several links, and the previous version of an entity. Windows, aggregates over children, `previous` and multi-link paths are flux-expr/2 extensions that a FLUX bundle may use. The rest waits for FLUID SQL/2. @@ -387,10 +394,21 @@ Consumers render SQL from the parsed expression, never from the raw text (see Se | `ROUND(x, s, 'HALF_TO_EVEN')` | Snowflake `ROUND(x, s, 'HALF_TO_EVEN')`, Databricks `bround(x, s)`, elsewhere the `CASE` form below | DuckDB's `round_even` returns `DOUBLE`, so it is not used for `DECIMAL` | | `DAYS_BETWEEN(a, b)` | `b - a` (PostgreSQL), `DATEDIFF(day, a, b)` (Snowflake), `DATE_DIFF(b, a, DAY)` (BigQuery), `datediff(b, a)` (Databricks), `date_sub('day', a, b)` (DuckDB) | both are dates, so day boundaries and whole days agree | | `MONTHS_BETWEEN(a, b)`, `YEARS_BETWEEN(a, b)` | DuckDB `date_sub('month', a, b)` or `date_sub('year', a, b)`, elsewhere the `EXTRACT` form below | never `DATEDIFF`, `DATE_DIFF`, `date_diff` or an engine's own `MONTHS_BETWEEN` | +| `ADD_DAYS(d, n)` | DuckDB and PostgreSQL `d + CAST(k AS INTEGER)`, Snowflake `DATEADD(day, k, d)`, BigQuery `DATE_ADD(d, INTERVAL CAST(k AS INT64) DAY)`, Databricks `date_add(d, CAST(k AS INT))`, Athena (Trino) `date_add('day', k, d)`; `k` is the guarded count below | every engine adds whole days to a date | +| `ADD_MONTHS(d, n)` | DuckDB `CAST(d + to_months(CAST(k AS INTEGER)) AS DATE)` (`d + INTERVAL n MONTH`, back to a date), PostgreSQL `CAST(d + CAST(k AS INTEGER) * INTERVAL '1 month' AS DATE)`, Snowflake `DATEADD(month, k, d)`, BigQuery `DATE_ADD(d, INTERVAL CAST(k AS INT64) MONTH)`, Databricks `add_months(d, CAST(k AS INT))`, Athena `d + k * INTERVAL '1' MONTH` | these clamp the day as §2.6 does; never Snowflake's `ADD_MONTHS` (it keeps the month end) | +| `ADD_YEARS(d, n)` | as `ADD_MONTHS`, with `to_years`, `INTERVAL '1 year'`, `DATEADD(year, ...)`, `INTERVAL ... YEAR`, `add_months(d, CAST(k AS INT) * 12)` and `INTERVAL '1' YEAR` | 29 February clamps to 28 February | | a string compared with `<`, `<=`, `>` or `>=` | as written | PostgreSQL adds `COLLATE "C"`, so text orders by code point, as in Snowflake, BigQuery and DuckDB | | `rel.col` | `LEFT JOIN AS rel ON ` | a left join whatever the relationship's `joinType`, so a missing row reads as NULL | | `DATE 'YYYY-MM-DD'` | as written | SQL's typed literal | +The guarded count of a date move. Engines raise, or return a date past 9999, where FLUID SQL/1 gives NULL (§2.6). So the count is made NULL first, by a `CASE` that only selects, and the engine then adds what it selected: an engine that evaluates every branch of a `CASE` still never adds an out-of-range count. With `m = EXTRACT(YEAR FROM d) * 12 + EXTRACT(MONTH FROM d) - 1` and `y = EXTRACT(YEAR FROM d)`: + +- `ADD_DAYS`: `k = CASE WHEN n BETWEEN AND THEN n END`, the days counted with the engine's day difference above; +- `ADD_MONTHS`: `k = CASE WHEN n BETWEEN 12 - m AND 119999 - m THEN n END`; +- `ADD_YEARS`: `k = CASE WHEN n BETWEEN 1 - y AND 9999 - y THEN n END`. + +The bounds are computed from `d`, never from `n`, so a huge count overflows nothing. Three engines outside the rendered dialects need care. SQL Server's `DATEADD(month, k, d)` clamps, and raises outside its range, so it takes the guarded count too. Oracle's and Db2's `ADD_MONTHS` keep the month end, so neither is used: Db2 writes `d + k MONTHS` (a labeled duration, which clamps), and Oracle `LEAST(ADD_MONTHS(TRUNC(d, 'MM'), k) + (EXTRACT(DAY FROM d) - 1), LAST_DAY(ADD_MONTHS(TRUNC(d, 'MM'), k)))`, which moves the first of the month and clamps the day itself. A consumer that cannot write one of these forms refuses the text. sqlglot is not a way around this: 30.33 rewrites Oracle's and Snowflake's `ADD_MONTHS` faithfully only into DuckDB, and passes it to every other dialect by name, with the month end kept. + The `CASE` form of half-to-even, with `y = x * 10^s` in fixed point: `CASE WHEN ABS(y - TRUNC(y)) = 0.5 THEN TRUNC(y) + CASE WHEN MOD(TRUNC(y), 2) = 0 THEN 0 ELSE SIGN(y) END ELSE ROUND(y) END / 10^s`. @@ -423,6 +441,7 @@ flux-expr/2 is spelled in SQL (maintainers' decision, 2026-11-16, in Agenticstig | `ROUND(x, s, 'HALF_TO_EVEN')` | #38's half-even `round(x, s)` | | `ROUND(x, s)` | rounding to s places with ties away from zero (change 2) | | `DAYS_BETWEEN`, `MONTHS_BETWEEN`, `YEARS_BETWEEN` | `days_between`, `months_between`, `years_between` | +| `ADD_DAYS`, `ADD_MONTHS`, `ADD_YEARS` | flux-expr/3's `ADD_DAYS`, `ADD_MONTHS` and `ADD_YEARS` (EXPR-SPEC-3 §4), the same three calls; ADR 0013 makes them core once FLUID SQL has them | | `DATE '2026-11-16'` | `date('2026-11-16')` | | a column name `c` | the explicit path `row.c`, so a column named `row`, `link`, `previous` or `children`, or one that shares a relationship's name, needs no rule of its own; a quoted name that flux-expr/2's identifiers cannot spell is a FLUX refusal until flux-expr/2 can quote it | | `rel.col` | `link..col`, always through the `link` root (EXPR-SPEC-2 §5), so a relationship named `row`, `link`, `previous` or `children`, or one that shares a column's name, still reads the linked row | @@ -668,6 +687,7 @@ Each step is its own PR: - dbt constraints and data tests: docs.getdbt.com/reference/resource-properties/constraints, /reference/data-test-configs; dbt_utils `expression_is_true`; dbt-expectations, maintained at github.com/metaplane/dbt-expectations - Snowflake `ROUND`, `MONTHS_BETWEEN`, `DATEDIFF`, `IS [NOT] DISTINCT FROM`, `EQUAL_NULL` and semantic view validation rules: docs.snowflake.com/en/sql-reference/functions/, /user-guide/views-semantic/validation-rules - BigQuery date functions and rounding modes: docs.cloud.google.com/bigquery/docs/reference/standard-sql/date_functions; DuckDB date functions: duckdb.org/docs/2.0/sql/functions/date.html +- Month arithmetic at the month end (§2.6, §2.8): PostgreSQL postgresql.org/docs/current/functions-datetime.html; Snowflake docs.snowflake.com/en/sql-reference/functions/dateadd and /add_months; Spark spark.apache.org/docs/latest/api/sql/ and its 3.0 migration guide (`add_months` stopped keeping the month end); Trino trino.io/docs/current/functions/datetime.html; SQL Server learn.microsoft.com/en-us/sql/t-sql/functions/dateadd-transact-sql; Oracle docs.oracle.com/en/database/oracle/oracle-database/19/sqlrf/ADD_MONTHS.html; Db2 ibm.com/docs/en/db2/11.5?topic=functions-add-months and its date arithmetic (labeled durations); flux-engine ADR 0013 and EXPR-SPEC-3 - Apache Ossie core spec: github.com/apache/ossie - Deequ `satisfies` and `Compliance`: github.com/awslabs/deequ - J. Kivinen and H. Mannila, "Approximate inference of functional dependencies from relations", Theoretical Computer Science 149(1), 1995; Y. Huhtala et al., "TANE", The Computer Journal 42(2), 1999 diff --git a/fluid_build/cli/bootstrap.py b/fluid_build/cli/bootstrap.py index 73a2f557..5564b664 100644 --- a/fluid_build/cli/bootstrap.py +++ b/fluid_build/cli/bootstrap.py @@ -513,7 +513,8 @@ def _cmd(name: str, help_text: Any, *aliases: str) -> Tuple[str, Any, Tuple[str, ( _cmd( "profile", - "Profile an expose's records where they live (aggregates only, output-checked)", + "Profile a data product's records where they live " + "(aggregates only, output-checked)", ), ), ), diff --git a/fluid_build/cli/profile_cmd.py b/fluid_build/cli/profile_cmd.py index 06dac030..fe93f97e 100644 --- a/fluid_build/cli/profile_cmd.py +++ b/fluid_build/cli/profile_cmd.py @@ -1,10 +1,10 @@ # Copyright 2025 Agentics Transformation Limited # SPDX-License-Identifier: Apache-2.0 -"""``fluid profile``: an output-checked profile of one expose, read where its records live. +"""``fluid profile``: an output-checked profile of a data product, read where its records live. -The command reads the expose's records once, inside the environment that holds -them, and writes a FLUX profile document (PROFILE-SPEC format 1-0-0, or 1-1-0 +The command reads each expose's records once, inside the environment that holds +them, and writes one FLUX profile document (PROFILE-SPEC format 1-0-0, or 1-1-0 when it holds a quantile with bins) of aggregates that passed the disclosure rules. A FLUX Simulation pins the document by the digest this prints. See :mod:`fluid_build.profiler` for the parts. @@ -20,12 +20,15 @@ import logging import re from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Sequence, Tuple COMMAND = "profile" _DESCRIPTION = """\ -Write an output-checked profile of one expose's records, read where they live. +Write an output-checked profile of an expose's records, read where they live. +Several exposes of the data product go into one document: give each one its +records as --source = (and a database's table as --table +=
); the document is then named after the contract's id. The document follows FLUX PROFILE-SPEC (format 1-0-0, or 1-1-0 when it holds a quantile with bins) and holds only aggregates that passed the output checks: no @@ -49,6 +52,26 @@ sum on a grid of 2^-N steps. N is --fractional-bits, 16 unless given, and the fidelity block's encoding.fractionalBits must be the same. +Without --bundle, the document also carries the statistics the contract's own +declarations give, declared from the contract and never from the records, as the +fidelity block FLUX's grammar reads (so `fluxsim init` can start from them): + - for each enum(...) column, a levelHistogram over every declared level; a + level no record holds is a withheld cell, and the column's distinctCount + is withheld (D); + - for a number whose validationRules declare both ends of a range inside + its type, a marginalHistogram over about ten bins of it; its sum, min, + max, quantiles and distinctCount are then withheld (D), and it has no + mean/1; + - for any other number of a bounded type, and each date and timestamp, + quantile/2 at 0.25, 0.5 and 0.75 on a ladder fixed by the type (0, 1, 2, + 5, 10, 20, 50 ... for a number, one bin per year from 1900 to 2100 for a + date or a timestamp); + - for each categorical semantics.dimensions entry on an enum column, the + other enum columns' histograms per level of it, every declared level a + group, up to 10,000 cells for one column's groupings. +A record outside what the contract declares (a level its type does not list, a +value outside its type or its declared range) refuses the profile. + With --bundle DIR the profile answers the Simulation's fidelity block instead: it reads the block of DIR's one Simulation (*.flux.yml) for the expose and writes the statistic each declared property is compared with (a mean, a @@ -60,10 +83,11 @@ group count as one more group there. A correlation is not answered: its moments need the exact record count, which the output checks round. -What the checks leave readable: a grouped statistic lists every group the data -forms, published or withheld. A key is always a level the bundle declares (a -value outside the declared support is refused), so its presence says only that -the level holds at least one record. +What the checks leave readable: under --bundle, a grouped statistic lists every +group the data forms, published or withheld. A key is always a level the bundle +declares (a value outside the declared support is refused), so its presence says +only that the level holds at least one record. Without --bundle, every declared +level is a group, whatever the records hold. The counts a histogram, a level histogram or a quantile publishes at the edges of one column are checked together, as FLUX ADR 0012 sets, by flux-stats' @@ -79,9 +103,9 @@ formed by adding or subtracting what they publish, so more is withheld (D in the summary). Only then are counts rounded. The check does not see a column's support, which lists the levels that hold records, or the group sizes of a -statistic grouped by the column, which repeat its level counts; and a declared -level that holds no record is absent from a grouped statistic, which shows its -zero. A published min or +statistic grouped by the column, which repeat its level counts; and under +--bundle a declared level that holds no record is absent from a grouped +statistic, which shows its zero. A published min or max shows that no record lies below or above it. A quantile whose value is the column's smallest or largest shows the same, and is withheld for it, as a zero is. @@ -98,10 +122,13 @@ # a database table (the URI's password is never printed) fluid profile contract.fluid.yaml --expose orders \\ --source postgresql://db.internal:5432/sales --table public.orders +# the whole data product in one document: one --source = each +fluid profile contract.fluid.yaml --source customers=./customers.csv \\ + --source orders=./orders.parquet --out product.profile.json # Only aggregates leave: a count under 10 or a zero is withheld, counts are rounded # to 5, a sum 2 records dominate is withheld, and a min, max or quantile is shown only -# when 10 records hold it (ACRO 1.1.0 defaults). With --bundle, a column's histograms and +# when 10 records hold it (ACRO 1.1.0 defaults). A column's histograms and # quantiles are checked together (FLUX ADR 0012): on one partition, a quantile with bins # only with at least 10 records below its bin, in it and above it, and no small cell, # a zero included, left to be formed by adding or subtracting what is published. A group @@ -125,7 +152,7 @@ def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") """Wire ``fluid profile`` into the CLI.""" p = subparsers.add_parser( COMMAND, - help="Profile an expose's records where they live (aggregates only, output-checked)", + help="Profile a data product's records where they live (aggregates only, output-checked)", description=_DESCRIPTION, epilog=_EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter, @@ -138,23 +165,39 @@ def register(subparsers: "argparse._SubParsersAction[argparse.ArgumentParser]") "--expose", "--expose-id", dest="expose_id", - help="exposeId to profile (default: the contract's only expose)", + # Collected, not overwritten: argparse would keep the last of two and read the + # first expose's source as the second one. + action="append", + help=( + "exposeId to profile; repeat it to profile several exposes into one document, " + "each with its own --source = (default: the contract's only " + "expose, or every expose a --source names)" + ), ) p.add_argument( "--source", required=True, + action="append", help=( "where the records live: a .ndjson, .jsonl, .csv or .parquet file (a path " - "or a file:// URL), or a postgres://, mysql:// or sqlite:/// database URI" + "or a file:// URL), or a postgres://, mysql:// or sqlite:/// database URI; " + "for several exposes, one --source = each" ), ) p.add_argument( "--table", - help="the table of a database --source, as name or schema.name", + action="append", + help=( + "the table of a database --source, as name or schema.name; for several " + "exposes, --table =
" + ), ) p.add_argument( "--out", - help="where to write the profile (default: .profile.json here)", + help=( + "where to write the profile (default: .profile.json here, or " + ".profile.json for several exposes)" + ), ) p.add_argument( "--fractional-bits", @@ -231,18 +274,25 @@ def run(args: argparse.Namespace, logger: logging.Logger) -> int: bundle_dir = _bundle_dir(args.bundle) contract = load_contract_with_overlay(str(contract_path), args.env, logger) try: - expose_id = runner.resolve_expose_id(contract, args.expose_id) - out = _out_path(args.out, expose_id, args.source) - block = bundle.read_bundle_block(bundle_dir, expose_id, backend) if bundle_dir else None - result = runner.profile_expose( + exposes = _expose_sources(contract, args.expose_id, args.source, args.table) + out = _out_path( + args.out, _document_name(contract, exposes), [each.source for each in exposes] + ) + blocks = ( + { + each.expose_id: bundle.read_bundle_block(bundle_dir, each.expose_id, backend) + for each in exposes + } + if bundle_dir + else None + ) + result = runner.profile_exposes( contract, - expose_id, - source=args.source, - table=args.table, + exposes, cwd=Path.cwd(), backend=backend, fractional_bits=args.fractional_bits, - bundle=block, + bundles=blocks, ) _write(out, result.canonical) except ( @@ -256,10 +306,164 @@ def run(args: argparse.Namespace, logger: logging.Logger) -> int: _stats.DocumentRefused, ) as exc: raise _as_refusal(exc, args.source) from None - _report(cprint_literal, result, out, answered_block=block is not None) + _report(cprint_literal, result, out, answered_block=blocks is not None) return 0 +def _paired(contract_ids: Sequence[str], value: str) -> Tuple[Optional[str], str]: + """``(exposeId, rest)`` for ``=``, else ``(None, value)``. + + A value is paired only when the text before its first ``=`` is an exposeId of the + contract, so a database URI's query (``?sslmode=require``) or a path holding ``=`` + stays whole; ``./orders=2024.csv`` reads a file named ``orders=2024.csv``. + """ + head, sep, rest = value.partition("=") + if sep and head in contract_ids: + return head, rest + return None, value + + +def _expose_sources( + contract: Any, + exposes: Optional[List[str]], + given_sources: List[str], + given_tables: Optional[List[str]], +) -> List[Any]: + """Pair each expose to profile with its ``--source`` (and ``--table``), before any read. + + One expose takes a bare ``--source``: the one ``--expose``, or the contract's only + expose. Several take one ``--source =`` each, and are the + ``--expose`` flags given, in that order, or else every expose a source names, in the + contract's order. Every refusal names a database source with its password masked. + """ + from fluid_build.profiler import runner + from fluid_build.profiler.sources import describe_source + + ids = runner.expose_ids(contract) + named = list(exposes or []) + for position, requested in enumerate(named): + if requested in named[:position]: + raise runner.ProfileError( + "profile_expose_repeated", + f"--expose names {requested!r} twice", + {"read": "nothing: no record was read", "next": "name each expose once"}, + ) + if requested not in ids: + runner.resolve_expose_id(contract, requested) # profile_expose_not_found + pairs = [_paired(ids, value) for value in given_sources] + tables = [_paired(ids, value) for value in given_tables or []] + if all(expose_id is None for expose_id, _ in pairs): + if len(pairs) > 1 or len(named) > 1: + raise runner.ProfileError( + "profile_source_unpaired", + ( + f"--source is given {len(pairs)} times" + if len(pairs) > 1 + else f"--expose is given {len(named)} times ({', '.join(named)})" + ) + + ", and no --source names its expose", + { + "read": "nothing: no record was read", + "next": "give each expose its records as --source =", + }, + ) + only = runner.resolve_expose_id(contract, named[0] if named else None) + return [runner.ExposeSource(only, pairs[0][1], _one_table(tables, [only]))] + for paired, location in pairs: + if paired is None: + raise runner.ProfileError( + "profile_source_unpaired", + f"--source {describe_source(location)} names no expose of the contract", + { + "read": "nothing: no record was read", + "exposes": ", ".join(ids), + "next": "with several sources, write each as --source =", + }, + ) + by_expose: Dict[str, str] = {} + for paired, location in pairs: + if str(paired) in by_expose: + raise runner.ProfileError( + "profile_source_repeated", + f"exposes[{paired}] is given more than one --source", + {"read": "nothing: no record was read", "next": "give each expose one source"}, + ) + by_expose[str(paired)] = location + if named: + missing = [expose_id for expose_id in named if expose_id not in by_expose] + if missing: + raise runner.ProfileError( + "profile_source_missing", + f"exposes[{missing[0]}] has no --source {missing[0]}=", + {"read": "nothing: no record was read", "next": "give every --expose its source"}, + ) + unused = [expose_id for expose_id in by_expose if expose_id not in named] + if unused: + raise runner.ProfileError( + "profile_source_unused", + f"--source names exposes[{unused[0]}], which no --expose names", + { + "read": "nothing: no record was read", + "next": f"add --expose {unused[0]}, or leave its --source out", + }, + ) + order = named + else: + order = [expose_id for expose_id in ids if expose_id in by_expose] + chosen = _tables_by_expose(tables, order) + return [ + runner.ExposeSource(expose_id, by_expose[expose_id], chosen.get(expose_id)) + for expose_id in order + ] + + +def _one_table(tables: List[Tuple[Optional[str], str]], exposes: List[str]) -> Optional[str]: + """The ``--table`` of a profile of one expose: bare, or paired with that expose.""" + if not tables: + return None + return _tables_by_expose(tables, exposes, bare=True).get(exposes[0]) + + +def _tables_by_expose( + tables: List[Tuple[Optional[str], str]], exposes: List[str], *, bare: bool = False +) -> Dict[str, str]: + """Each ``--table`` by the expose it names; a bare one only where *bare* allows one.""" + from fluid_build.profiler.runner import ProfileError + + found: Dict[str, str] = {} + for expose_id, table in tables: + if expose_id is None: + if not bare or len(tables) > 1: + raise ProfileError( + "profile_table_unpaired", + "--table names no expose, and several tables or exposes are given", + {"next": "write each as --table =
"}, + ) + expose_id = exposes[0] + if expose_id not in exposes: + raise ProfileError( + "profile_table_unused", + f"--table names exposes[{expose_id}], which this profile does not read", + {"next": "name a table for an expose that is profiled"}, + ) + if expose_id in found: + raise ProfileError( + "profile_table_repeated", + f"exposes[{expose_id}] is given more than one --table", + {"next": "give each expose one table"}, + ) + found[expose_id] = table + return found + + +def _document_name(contract: Any, exposes: List[Any]) -> str: + """The default document name: the expose's id, or for several the contract's ``id``.""" + if len(exposes) == 1: + return str(exposes[0].expose_id) + product = contract.get("id") if hasattr(contract, "get") else None + return str(product) if isinstance(product, str) else "" + + def _bundle_dir(given: Optional[str]) -> Optional[Path]: """``--bundle``, checked as the contract is, and a directory that exists.""" from fluid_build.cli.security import validate_cli_path @@ -284,7 +488,18 @@ def _bundle_dir(given: Optional[str]) -> Optional[Path]: return path -def _as_refusal(exc: Exception, source: str) -> Any: +def _described(given_sources: Sequence[str]) -> str: + """Every ``--source`` as it may be shown: each database URI with its password masked. + + :func:`~fluid_build.profiler.sources.describe_source` masks a URI after its + ``=`` too, a mistyped exposeId included. + """ + from fluid_build.profiler.sources import describe_source + + return ", ".join(describe_source(value) for value in given_sources) + + +def _as_refusal(exc: Exception, given_sources: Sequence[str]) -> Any: """The :class:`CLIError` for a refusal from the profiler; nothing has been written.""" from fluid_build.profiler import _stats, bundle, runner, sources @@ -309,7 +524,7 @@ def _as_refusal(exc: Exception, source: str) -> Any: return _refusal( "profile_source_unreadable", str(exc), - {"source": sources.describe_source(source), "written": "nothing"}, + {"source": _described(given_sources), "written": "nothing"}, ) if isinstance(exc, _stats.ReadRefused): return _refusal( @@ -336,8 +551,10 @@ def _as_refusal(exc: Exception, source: str) -> Any: _PLAIN_NAME = re.compile(r"[A-Za-z0-9_.-]+\Z") -def _out_path(given: Optional[str], expose_id: str, source: str) -> Path: - """Where the document goes: ``--out``, or ``.profile.json`` here. +def _out_path(given: Optional[str], name: str, given_sources: Sequence[str]) -> Path: + """Where the document goes: ``--out``, or ``.profile.json`` here. + + *name* is the one expose's id, or for several exposes the contract's ``id``. Checked before any record is read, and without touching the file system: no directory is made and nothing is written until the profile is formed @@ -349,16 +566,15 @@ def _out_path(given: Optional[str], expose_id: str, source: str) -> Path: replaced by their own profile. """ from fluid_build.profiler.runner import ProfileError - from fluid_build.profiler.sources import local_file if given: target = Path(given) - elif _PLAIN_NAME.match(expose_id) and expose_id not in (".", ".."): - target = Path(f"{expose_id}.profile.json") + elif _PLAIN_NAME.match(name) and name not in (".", ".."): + target = Path(f"{name}.profile.json") else: raise ProfileError( "profile_out_required", - f"exposeId {expose_id!r} cannot name a file", + f"{name!r} cannot name a file" if name else "the contract has no id to name it by", {"next": "name the document with --out"}, ) unresolved = target if target.is_absolute() else Path.cwd() / target @@ -369,14 +585,7 @@ def _out_path(given: Optional[str], expose_id: str, source: str) -> Path: f"{unresolved} is a symbolic link", {"next": "remove the link, or name another file with --out"}, ) - read_from = local_file(source) - if read_from is not None and read_from.exists() and unresolved.exists(): - if unresolved.samefile(read_from): - raise ProfileError( - "profile_out_is_the_source", - f"{unresolved} is the file the records are read from", - {"next": "name another file with --out"}, - ) + _refuse_a_source(unresolved, given_sources) except OSError as exc: raise ProfileError( "profile_out_unusable", @@ -385,6 +594,23 @@ def _out_path(given: Optional[str], expose_id: str, source: str) -> Path: return unresolved +def _refuse_a_source(target: Path, given_sources: Sequence[str]) -> None: + """Refuse a document target that is a file the records are read from.""" + from fluid_build.profiler.runner import ProfileError + from fluid_build.profiler.sources import local_file + + if not target.exists(): + return + for source in given_sources: + read_from = local_file(source) + if read_from is not None and read_from.exists() and target.samefile(read_from): + raise ProfileError( + "profile_out_is_the_source", + f"{target} is the file the records are read from", + {"next": "name another file with --out"}, + ) + + def _write(out: Path, data: bytes) -> None: """Check the target as every ``--out`` is, then write the document owner-only and atomically. @@ -430,13 +656,13 @@ def _codes(counts: Dict[str, int]) -> str: ) -def _one_code_withheld(result: Any) -> bool: - """Whether a histogram or a quantile, whose withheld values all read C, has one.""" +def _one_code_withheld(table: Any) -> bool: + """Whether a table's histogram or quantile, whose withheld values all read C, has one.""" from fluid_build.profiler.disclosure import ONE_CODE return any( each[1].partition("/")[0] in ONE_CODE - for each in (*result.withheld_statistics, *result.partly_withheld_statistics) + for each in (*table.withheld_statistics, *table.partly_withheld_statistics) ) @@ -471,58 +697,80 @@ def _named_partly(statistics: Any) -> List[str]: def _report(say: Any, result: Any, out: Path, *, answered_block: bool = False) -> None: - """Tell the owner what the document holds, then print the pin.""" - plan = result.plan + """Tell the owner what the document holds, then print the pin. + + A profile of several exposes names each line about a table by its expose. + """ + tables = result.tables + several = len(tables) > 1 withheld: Dict[str, int] = dict(result.withheld) - say( - f"Profiled exposes[{plan.expose_id}]: {_plural(len(plan.columns), 'column')}, " - f"{_plural(result.published, 'value')} published, {sum(withheld.values())} withheld." - ) + counts = f"{_plural(result.published, 'value')} published, {sum(withheld.values())} withheld." + if several: + columns = sum(len(table.plan.columns) for table in tables) + names = ", ".join(table.plan.expose_id for table in tables) + say( + f"Profiled {len(tables)} exposes ({names}): {_plural(columns, 'column')}, {counts}", + soft_wrap=True, + ) + else: + plan = tables[0].plan + say(f"Profiled exposes[{plan.expose_id}]: {_plural(len(plan.columns), 'column')}, {counts}") if withheld: say(f"Withheld, by the check (CL_CONF_STATUS code): {_codes(withheld)}.") cells: Dict[str, int] = dict(result.withheld_cells) if cells: say(f"Withheld histogram cells, by the check (CL_CONF_STATUS code): {_codes(cells)}.") - if cells or _one_code_withheld(result): + if cells or any(_one_code_withheld(table) for table in tables): say( "The document writes C for every withheld group and cell of a histogram or " "a quantile, whatever withheld it, so its code says nothing of its size.", soft_wrap=True, ) - named = _named(result.statistics) - whole = _named(result.withheld_statistics) + grid = ( + f"on the fidelity block's fractionalBits {result.fractional_bits}" + if answered_block + else f"on fractionalBits {result.fractional_bits} " + "(the fidelity block must declare the same)" + ) + for table in tables: + _report_table(say, table, f"exposes[{table.plan.expose_id}]: " if several else "", grid) + if not result.published: + say("Every value was withheld: the document says nothing about the records.") + say(f"Wrote {out} (readable by you alone; read it before you share it).", soft_wrap=True) + say("Pin it in the FLUX Simulation:") + # One line, never wrapped: it is copied as it stands. + say(f" profile: {{digest: {result.digest}}}", soft_wrap=True) + + +def _report_table(say: Any, table: Any, prefix: str, grid: str) -> None: + """The summary lines about one table, each led by *prefix* (its expose, or nothing).""" + named = _named(table.statistics) + whole = _named(table.withheld_statistics) if named: - grid = ( - f"on the fidelity block's fractionalBits {result.fractional_bits}" - if answered_block - else f"on fractionalBits {result.fractional_bits} " - "(the fidelity block must declare the same)" - ) - say(f"Fidelity statistics: {'; '.join(named)}, {grid}.", soft_wrap=True) + say(f"{prefix}Fidelity statistics: {'; '.join(named)}, {grid}.", soft_wrap=True) if whole: say( - "Withheld whole, so a run that checks them scores them censored: " + f"{prefix}Withheld whole, so a run that checks them scores them censored: " f"{'; '.join(whole)}.", soft_wrap=True, ) - partly = _named_partly(result.partly_withheld_statistics) + partly = _named_partly(table.partly_withheld_statistics) if partly: say( - "Partly withheld, so a run that checks them scores only the published groups: " - f"{'; '.join(partly)}.", + f"{prefix}Partly withheld, so a run that checks them scores only the published " + f"groups: {'; '.join(partly)}.", soft_wrap=True, ) - for description, reason in result.left_out: + for description, reason in table.left_out: say( - f"Not answered: {description}: {reason}; a run that checks it is refused " + f"{prefix}Not answered: {description}: {reason}; a run that checks it is refused " "(FLX-5037).", soft_wrap=True, ) - for name, reason in plan.skipped: - say(f"Not profiled: column {name!r}: {reason}.") - if not result.published: - say("Every value was withheld: the document says nothing about the records.") - say(f"Wrote {out} (readable by you alone; read it before you share it).", soft_wrap=True) - say("Pin it in the FLUX Simulation:") - # One line, never wrapped: it is copied as it stands. - say(f" profile: {{digest: {result.digest}}}", soft_wrap=True) + for name, reason in table.plan.skipped: + say(f"{prefix}Not profiled: column {name!r}: {reason}.") + for name, reason in table.uncovered: + say( + f"{prefix}Left out of the default statistics: column {name!r}: {reason}.", + soft_wrap=True, + ) diff --git a/fluid_build/fluid_sql/_canonical.py b/fluid_build/fluid_sql/_canonical.py index daabe564..538d6b28 100644 --- a/fluid_build/fluid_sql/_canonical.py +++ b/fluid_build/fluid_sql/_canonical.py @@ -123,4 +123,7 @@ def _bare(node: n.Node) -> str: if isinstance(node, n.DateDiff): name = n.DATE_DIFF_FUNCTIONS[node.unit] return f"{name}({_text(node.from_, _ADD)}, {_text(node.to, _ADD)})" + if isinstance(node, n.DateAdd): + name = n.DATE_ADD_FUNCTIONS[node.unit] + return f"{name}({_text(node.date, _ADD)}, {_text(node.count, _ADD)})" raise TypeError(f"not a FLUID SQL/1 node: {node!r}") diff --git a/fluid_build/fluid_sql/_checker.py b/fluid_build/fluid_sql/_checker.py index b5f42f0d..ffbe696d 100644 --- a/fluid_build/fluid_sql/_checker.py +++ b/fluid_build/fluid_sql/_checker.py @@ -18,6 +18,10 @@ a comparison or ``BETWEEN``, ``IS [NOT] NULL``, unary minus, ``ROUND``'s argument, a predicate position, a whole value) it is refused, and so is a construct whose operands are all NULL literals. +- ``ADD_DAYS``, ``ADD_MONTHS`` and ``ADD_YEARS`` take a date and a number of scale + 0 (a float, a decimal without ``(p,s)`` and a ``/`` have no static scale), and + neither argument may be the NULL literal: it would give the call no type to + check the other against, and the count's scale could not be known. - A string literal compared with a column that declares an enum is one of its values, or one of the column's placeholders. - Scales: a literal has its written scale and a column its declared one; ``+`` @@ -353,8 +357,31 @@ def _typed(self, node: n.Node) -> _T: # noqa: C901 - one branch per construct if t.kind is not None: self._need(t, DATE, arg, name) return _T(NUMBER, 0) + if isinstance(node, n.DateAdd): + return self._date_add(node) raise TypeError(f"not a FLUID SQL/1 node: {node!r}") # pragma: no cover + def _date_add(self, node: n.DateAdd) -> _T: + """``ADD_DAYS``, ``ADD_MONTHS`` or ``ADD_YEARS``: a date and a count of scale 0.""" + name = n.DATE_ADD_FUNCTIONS[node.unit] + date, count = self.typed(node.date), self.typed(node.count) + for t, arg in ((date, node.date), (count, node.count)): + if t.kind is None: + raise FluidSqlError( + "null-literal", f"{name} takes a value, not the NULL literal", arg.pos + ) + self._need(date, DATE, node.date, name) + self._need(count, NUMBER, node.count, name) + if count.scale != 0: + raise FluidSqlError( + "type-mismatch", + f"{name} takes a count of scale 0, but {_short(node.count)} has " + + ("no static scale" if count.scale is None else f"scale {count.scale}") + + " (ROUND(x, 0) makes a whole count)", + node.count.pos, + ) + return _T(DATE) + def _need(self, t: _T, kind: str, node: n.Node, construct: str) -> None: if t.kind != kind: article = "an" if kind[0] in "aeiou" else "a" diff --git a/fluid_build/fluid_sql/_errors.py b/fluid_build/fluid_sql/_errors.py index 5a951962..6a12047b 100644 --- a/fluid_build/fluid_sql/_errors.py +++ b/fluid_build/fluid_sql/_errors.py @@ -30,7 +30,7 @@ "type-mismatch": "operands or arguments of the wrong type, or a predicate position " "given something other than a bool", "null-literal": "the NULL literal where nothing gives it a type: an IN item, an operand " - "of a comparison or BETWEEN, or every operand of a construct", + "of a comparison or BETWEEN, an argument of a date move, or every operand of a construct", "enum-literal": "a string literal compared with a column whose enum does not list it", "derive-target": "a derive target with no FLUID SQL/1 type, a float, or a number " "without a declared scale", diff --git a/fluid_build/fluid_sql/_evaluate.py b/fluid_build/fluid_sql/_evaluate.py index cd975c82..6f27edfa 100644 --- a/fluid_build/fluid_sql/_evaluate.py +++ b/fluid_build/fluid_sql/_evaluate.py @@ -17,6 +17,12 @@ forward by months keeps the day of the month or lands on a shorter month's last day; from a later date to an earlier one the result is negative. +- ``ADD_DAYS(d, n)`` moves ``d`` by ``n`` days and ``ADD_MONTHS(d, n)`` by ``n`` months, + keeping the day of the month or clamping it to a shorter month's last day (no + month-end preservation: 2001-02-28 plus a month is 2001-03-28); ``ADD_YEARS(d, n)`` is + ``ADD_MONTHS(d, 12 * n)``. A result before 0001-01-01 or after 9999-12-31 is NULL, for + every ``n``, however large (EXPR-SPEC-3). + A row maps column names to Python values (``None``, ``bool``, ``Fraction``, ``str``, ``datetime.date``, ``datetime.datetime``); ``links`` maps each relationship to the row it reaches, or ``None`` when it reaches none. @@ -178,8 +184,40 @@ def value(self, node: n.Node) -> Any: # noqa: C901 - one branch per construct if node.unit == "month": return Fraction(months_between(start, end)) return Fraction(years_between(start, end)) + if isinstance(node, n.DateAdd): + return self.date_add(node) raise TypeError(f"not a FLUID SQL/1 node: {node!r}") # pragma: no cover + def date_add(self, node: n.DateAdd) -> Optional[datetime.date]: + """``ADD_DAYS``, ``ADD_MONTHS`` or ``ADD_YEARS``: NULL in, NULL out.""" + date, count = self.value(node.date), self.value(node.count) + if date is None or count is None: + return None + return add_to_date(date, node.unit, count) + + +def add_to_date(date: datetime.date, unit: str, count: Any) -> Optional[datetime.date]: + """``date`` moved by ``count`` days, months or years (``unit``), or ``None``. + + Total: a result outside ``date.min``..``date.max`` is ``None``, and so is a count that + is not a whole number (the checker only lets scale 0 through). + """ + count = Fraction(count) + if count.denominator != 1: + return None + k = count.numerator + if unit == "day": + # Bound k first: a huge k would overflow timedelta before the range check. + if not (date.min - date).days <= k <= (date.max - date).days: + return None + return date + datetime.timedelta(days=k) + months = k * 12 if unit == "year" else k + # Bound the month index first: a huge k would make add_months build a year past 9999. + index = date.year * 12 + date.month - 1 + months + if not 12 <= index <= 9999 * 12 + 11: + return None + return add_months(date, months) + def _compare(op: str, left: Any, right: Any) -> bool: if left is None or right is None: diff --git a/fluid_build/fluid_sql/_lexer.py b/fluid_build/fluid_sql/_lexer.py index 60e63ac5..68b1d133 100644 --- a/fluid_build/fluid_sql/_lexer.py +++ b/fluid_build/fluid_sql/_lexer.py @@ -63,7 +63,16 @@ ) #: The functions of FLUID SQL/1. Not keywords: a column may be called ``round``. -FUNCTIONS = ("ROUND", "COALESCE", "DAYS_BETWEEN", "MONTHS_BETWEEN", "YEARS_BETWEEN") +FUNCTIONS = ( + "ROUND", + "COALESCE", + "DAYS_BETWEEN", + "MONTHS_BETWEEN", + "YEARS_BETWEEN", + "ADD_DAYS", + "ADD_MONTHS", + "ADD_YEARS", +) _WHITESPACE = frozenset(" \t\n") _WORD_START = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_") diff --git a/fluid_build/fluid_sql/_nodes.py b/fluid_build/fluid_sql/_nodes.py index 7622d77d..eca5fc74 100644 --- a/fluid_build/fluid_sql/_nodes.py +++ b/fluid_build/fluid_sql/_nodes.py @@ -29,6 +29,7 @@ ``round`` ``arg``, ``scale`` (its digits, a string), ``mode`` (``HALF_AWAY_FROM_ZERO`` or ``HALF_TO_EVEN``) ``date_diff`` ``unit`` (``day``, ``month`` or ``year``), ``from``, ``to`` +``date_add`` ``unit`` (``day``, ``month`` or ``year``), ``date``, ``count`` ================= ===================================================== Parentheses leave no node. Positions (``pos``, an index into the NFC text) and @@ -242,6 +243,21 @@ def children(self) -> Iterator[Node]: DATE_DIFF_FUNCTIONS = {"day": "DAYS_BETWEEN", "month": "MONTHS_BETWEEN", "year": "YEARS_BETWEEN"} +@dataclass(frozen=True) +class DateAdd(Node): + unit: str # "day", "month" or "year" + date: Node + count: Node + + def children(self) -> Iterator[Node]: + yield self.date + yield self.count + + +#: The function name of each date addition, by unit. +DATE_ADD_FUNCTIONS = {"day": "ADD_DAYS", "month": "ADD_MONTHS", "year": "ADD_YEARS"} + + def walk(node: Node) -> Iterator[Node]: """``node`` and every node inside it, parents first, in text order.""" stack = [node] @@ -340,6 +356,13 @@ def to_json(node: Node) -> Dict[str, Any]: "from": to_json(node.from_), "to": to_json(node.to), } + if isinstance(node, DateAdd): + return { + "node": "date_add", + "unit": node.unit, + "date": to_json(node.date), + "count": to_json(node.count), + } raise TypeError(f"not a FLUID SQL/1 node: {node!r}") @@ -378,6 +401,7 @@ def from_json(data: Any) -> Node: "coalesce": lambda d: Coalesce(tuple(from_json(a) for a in d["args"])), "round": lambda d: Round(from_json(d["arg"]), int(d["scale"]), d["mode"]), "date_diff": lambda d: DateDiff(d["unit"], from_json(d["from"]), from_json(d["to"])), + "date_add": lambda d: DateAdd(d["unit"], from_json(d["date"]), from_json(d["count"])), } if kind not in builders: raise ValueError(f"unknown FLUID SQL/1 node {kind!r}") diff --git a/fluid_build/fluid_sql/_parser.py b/fluid_build/fluid_sql/_parser.py index 04907876..c4f7c324 100644 --- a/fluid_build/fluid_sql/_parser.py +++ b/fluid_build/fluid_sql/_parser.py @@ -75,6 +75,9 @@ "DATE_DIFF": "write DAYS_BETWEEN, MONTHS_BETWEEN or YEARS_BETWEEN (whole units)", "DATE_SUB": "write DAYS_BETWEEN, MONTHS_BETWEEN or YEARS_BETWEEN (whole units)", "TIMESTAMPDIFF": "write DAYS_BETWEEN, MONTHS_BETWEEN or YEARS_BETWEEN (whole units)", + "DATEADD": "write ADD_DAYS(d, n), ADD_MONTHS(d, n) or ADD_YEARS(d, n)", + "DATE_ADD": "write ADD_DAYS(d, n), ADD_MONTHS(d, n) or ADD_YEARS(d, n)", + "TIMESTAMPADD": "write ADD_DAYS(d, n), ADD_MONTHS(d, n) or ADD_YEARS(d, n)", "CURRENT_DATE": "FLUID SQL/1 never reads the clock: use a column or a DATE literal", "CURRENT_TIMESTAMP": "FLUID SQL/1 never reads the clock: use a column or a DATE literal", "NOW": "FLUID SQL/1 never reads the clock: use a column or a DATE literal", @@ -84,7 +87,8 @@ "SELECT": "FLUID SQL/1 has no subqueries", "EXISTS": "FLUID SQL/1 has no subqueries", "TIMESTAMP": "FLUID SQL/1 has no timestamp literals", - "INTERVAL": "FLUID SQL/1 has no interval literals or date arithmetic", + "INTERVAL": "FLUID SQL/1 has no interval literals: write ADD_DAYS(d, n), ADD_MONTHS(d, n) " + "or ADD_YEARS(d, n)", "LIKE": "FLUID SQL/1 has no pattern matching", "ILIKE": "FLUID SQL/1 has no pattern matching", "RLIKE": "FLUID SQL/1 has no pattern matching", @@ -461,6 +465,12 @@ def call(self) -> n.Node: args.append(self.expr()) self.expect_op(")", "',' or ')' in COALESCE") node = self.make(n.Coalesce(tuple(args), pos=token.pos)) + elif name in _DATE_ADD_UNITS: + date = self.value() + self.expect_op(",", f"',' and the count of {name}: {name}(d, n)") + count = self.value() + self.expect_op(")", f"')': {name} takes a date and a count") + node = self.make(n.DateAdd(_DATE_ADD_UNITS[name], date, count, pos=token.pos)) else: unit = {"DAYS_BETWEEN": "day", "MONTHS_BETWEEN": "month", "YEARS_BETWEEN": "year"} start = self.value() @@ -548,6 +558,9 @@ def _check_date(text: str, pos: int) -> None: ) +_DATE_ADD_UNITS = {"ADD_DAYS": "day", "ADD_MONTHS": "month", "ADD_YEARS": "year"} + + def _function_message(name: str) -> str: if name in _HINTS: why = _HINTS[name] diff --git a/fluid_build/fluid_sql/_render.py b/fluid_build/fluid_sql/_render.py index aade0b0b..ce3c21dc 100644 --- a/fluid_build/fluid_sql/_render.py +++ b/fluid_build/fluid_sql/_render.py @@ -24,6 +24,12 @@ - ``DAYS_BETWEEN`` is each engine's day difference; ``MONTHS_BETWEEN`` and ``YEARS_BETWEEN`` are DuckDB's ``date_sub`` (whole units) and elsewhere the ``EXTRACT`` form of §2.8, never an engine's ``DATEDIFF`` (unit boundaries crossed); +- ``ADD_DAYS``, ``ADD_MONTHS`` and ``ADD_YEARS`` are the engine's date plus a count of + days, months or years, which clamps to a shorter month's last day (Snowflake's own + ``ADD_MONTHS`` keeps month ends, so it is ``DATEADD``). The count is first made NULL when + the result would leave 0001-01-01..9999-12-31 (an inner ``CASE`` that only selects), so + the engine never adds a count that overflows, even when it evaluates ``CASE`` branches + eagerly or the count is far beyond its integer type; - a string compared with ``<``, ``<=``, ``>`` or ``>=`` gets ``COLLATE "C"`` on PostgreSQL, so text orders by code point as in the other engines. @@ -287,6 +293,8 @@ def _value(self, node: n.Node) -> str: # noqa: C901 - one branch per construct return self.round(node) if isinstance(node, n.DateDiff): return self.date_diff(node) + if isinstance(node, n.DateAdd): + return self.date_add(node) raise TypeError(f"not a FLUID SQL/1 node: {node!r}") # pragma: no cover def branches(self, node: n.Node, values: List[n.Node]) -> List[str]: @@ -340,17 +348,21 @@ def round(self, node: n.Round) -> str: # -- date differences -------------------------------------------------------------- + def days(self, a: str, b: str) -> str: + """The whole days from ``a`` to ``b`` (both rendered dates).""" + return { + "duckdb": f"date_sub('day', {a}, {b})", + "snowflake": f"DATEDIFF(day, {a}, {b})", + "bigquery": f"DATE_DIFF({b}, {a}, DAY)", + "databricks": f"datediff({b}, {a})", + "postgres": f"({b} - {a})", + "athena": f"date_diff('day', {a}, {b})", + }[self.dialect] + def date_diff(self, node: n.DateDiff) -> str: a, b = self.value(node.from_), self.value(node.to) if node.unit == "day": - return { - "duckdb": f"date_sub('day', {a}, {b})", - "snowflake": f"DATEDIFF(day, {a}, {b})", - "bigquery": f"DATE_DIFF({b}, {a}, DAY)", - "databricks": f"datediff({b}, {a})", - "postgres": f"({b} - {a})", - "athena": f"date_diff('day', {a}, {b})", - }[self.dialect] + return self.days(a, b) if self.whole_units: return f"date_sub('{node.unit}', {a}, {b})" months = self.whole_months @@ -390,3 +402,53 @@ def div12(self, months: str) -> str: "postgres": f"DIV({months}, 12)", "athena": f"({months} / 12)", }[self.dialect] + + # -- date additions ---------------------------------------------------------------- + + def date_add(self, node: n.DateAdd) -> str: + d, k = self.value(node.date), self.value(node.count) + guard = self.in_range(node.unit, d, k) + # Trino adds a BIGINT, and coerces no DECIMAL count (a decimal(10,0) column) to one. + cast = {"bigquery": "INT64", "databricks": "INT", "athena": "BIGINT"}.get( + self.dialect, "INTEGER" + ) + whole = f"CAST({guard} AS {cast})" + unit = node.unit + if self.dialect == "duckdb": + if unit == "day": + return f"({d} + {whole})" + return f"CAST({d} + to_{unit}s({whole}) AS DATE)" + if self.dialect == "postgres": + if unit == "day": + return f"({d} + {whole})" + return f"CAST({d} + {whole} * INTERVAL '1 {unit}' AS DATE)" + if self.dialect == "snowflake": + return f"DATEADD({unit}, {guard}, {d})" + if self.dialect == "bigquery": + return f"DATE_ADD({d}, INTERVAL {whole} {unit.upper()})" + if self.dialect == "databricks": + if unit == "day": + return f"date_add({d}, {whole})" + return f"add_months({d}, {whole}{' * 12' if unit == 'year' else ''})" + if unit == "day": # athena (Trino) + return f"date_add('day', {whole}, {d})" + return f"({d} + {whole} * INTERVAL '1' {unit.upper()})" + + def in_range(self, unit: str, d: str, k: str) -> str: + """``k``, or NULL when ``d`` moved by ``k`` units leaves 0001-01-01..9999-12-31. + + Each bound is compared with ``k`` alone, never with a sum: ``k`` may be far beyond + any integer type, and a sum would overflow before the comparison.""" + year = f"EXTRACT(YEAR FROM {d})" + if unit == "day": + low = self.days(d, self.date_literal("0001-01-01")) + high = self.days(d, self.date_literal("9999-12-31")) + elif unit == "month": + index = f"({year} * 12 + EXTRACT(MONTH FROM {d}) - 1)" + low, high = f"(12 - {index})", f"(119999 - {index})" + else: + low, high = f"(1 - {year})", f"(9999 - {year})" + return f"(CASE WHEN {k} BETWEEN {low} AND {high} THEN {k} END)" + + def date_literal(self, text: str) -> str: + return f"DATE {sql_literal(self.dialect, text)}" diff --git a/fluid_build/fluid_sql/_render_cel.py b/fluid_build/fluid_sql/_render_cel.py index 5394b660..d10c7d50 100644 --- a/fluid_build/fluid_sql/_render_cel.py +++ b/fluid_build/fluid_sql/_render_cel.py @@ -300,6 +300,11 @@ class of what it sits beside); refuses what has no CEL form.""" f"it uses {n.DATE_DIFF_FUNCTIONS[node.unit]}, and a CEL rule has no exact date " "arithmetic" ) + if isinstance(node, n.DateAdd): + raise CelUntranslatable( + f"it uses {n.DATE_ADD_FUNCTIONS[node.unit]}, and a CEL rule has no exact date " + "arithmetic" + ) if isinstance(node, n.Date): raise CelUntranslatable("it holds a DATE literal, which a CEL rule cannot compare") raise TypeError(f"not a FLUID SQL/1 node: {node!r}") # pragma: no cover @@ -419,7 +424,7 @@ def value(self, node: n.Node, want: Optional[str]) -> _V: # noqa: C901 - one pe return self.case(node, want) if isinstance(node, n.Coalesce): return self.coalesce(node, want) - self.cls(node) # Round, DateDiff, Date: refused with the reason + self.cls(node) # Round, DateDiff, DateAdd, Date: refused with the reason raise TypeError(f"not a FLUID SQL/1 node: {node!r}") # pragma: no cover def branch_class(self, nodes: List[n.Node], outer: Optional[str]) -> str: diff --git a/fluid_build/fluid_sql/fluid-sql-vectors.json b/fluid_build/fluid_sql/fluid-sql-vectors.json index 4c707241..e783ea87 100644 --- a/fluid_build/fluid_sql/fluid-sql-vectors.json +++ b/fluid_build/fluid_sql/fluid-sql-vectors.json @@ -1,5 +1,5 @@ { - "vectorsVersion": "1.0.0", + "vectorsVersion": "1.1.0", "language": "FLUID SQL/1", "description": "FLUID SQL/1 conformance vectors (forge-cli RFC-data-rules.md section 2.9). They define the language: every implementation (forge-cli's validator, flux-expr/2) parses, checks and evaluates every vector and reaches its verdict, tree, canonical text, type, scale and values. Each vector names a model, one expose: models[name] = {columns: {name: {type, enum?, placeholders?: {values?, prefixes?}}}, relationships?: {name: {to: }}, validityWindow?: true}, where type is the FLUID column type as a contract writes it and validityWindow says the expose declares a validity window, so no path may reach it through a relationship. A vector also names a site: predicate (check, when, aggregate.filter: the text must be a bool, and a NULL result counts as FALSE) or value (expr). With derive, the text is the expr of a derive on that column and must fit it. A refused vector gives one reason from reasonCodes and has exactly one defect. An accepted vector gives its tree (one node per construct; see nodes), its canonical text, its type and, for a number, its static scale (an integer, or unbounded), and eval cases: a row (column -> value), the links (relationship -> the row it reaches, or null for none; a relationship left out reaches none) and the result. A column left out of a row is NULL.", "encoding": "No JSON number appears anywhere in this file: numbers, scales and limits are strings. A number is a decimal string; a result of static scale s is written with exactly s digits after the point, and one of unbounded scale as its shortest decimal, or as p/q in lowest terms when no decimal is exact (1/3). Compare numbers by value. A bool is true or false, a string is itself (compared by code points, never normalized), a date is YYYY-MM-DD, a timestamp is ISO 8601 without an offset, a timestamp with time zone ISO 8601 with one (compare instants), and NULL is null. A float column's value is the shortest decimal that converts back to the double.", @@ -27,7 +27,10 @@ "COALESCE", "DAYS_BETWEEN", "MONTHS_BETWEEN", - "YEARS_BETWEEN" + "YEARS_BETWEEN", + "ADD_DAYS", + "ADD_MONTHS", + "ADD_YEARS" ], "limits": { "maxCodePoints": "4096", @@ -48,7 +51,7 @@ "validity-window": "a path through a relationship whose target declares a validity window", "untyped-column": "a column whose type has no FLUID SQL/1 type", "type-mismatch": "operands or arguments of the wrong type, or a predicate position given something other than a bool", - "null-literal": "the NULL literal where nothing gives it a type: an IN item, an operand of a comparison or BETWEEN, or every operand of a construct", + "null-literal": "the NULL literal where nothing gives it a type: an IN item, an operand of a comparison or BETWEEN, an argument of a date move, or every operand of a construct", "enum-literal": "a string literal compared with a column whose enum does not list it", "derive-target": "a derive target with no FLUID SQL/1 type, a float, or a number without a declared scale", "derive-type": "a derived value whose type differs from its target's", @@ -74,7 +77,8 @@ "case": "whens: [{when, then}], else when written", "coalesce": "args", "round": "arg, scale (its digits, a string), mode (HALF_AWAY_FROM_ZERO when none is written, or HALF_TO_EVEN)", - "date_diff": "unit (day, month, year), from, to" + "date_diff": "unit (day, month, year), from, to", + "date_add": "unit (day, month, year), date, count" }, "models": { "invoices": { @@ -190,6 +194,9 @@ }, "too_wide": { "type": "decimal(1001, 2)" + }, + "add_days": { + "type": "int" } }, "relationships": { @@ -5720,6 +5727,1528 @@ ] } }, + { + "id": "add-months-jan31-common-year", + "text": "ADD_MONTHS(DATE '2001-01-31', 1)", + "site": "value", + "model": "invoices", + "note": "EXPR-SPEC-3: the day is clamped to the target month's last day", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "2001-01-31" + }, + "count": { + "node": "number", + "value": "1" + } + }, + "canonical": "ADD_MONTHS(DATE '2001-01-31', 1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "2001-02-28" + } + ] + } + }, + { + "id": "add-months-jan31-leap-year", + "text": "ADD_MONTHS(DATE '2004-01-31', 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "2004-01-31" + }, + "count": { + "node": "number", + "value": "1" + } + }, + "canonical": "ADD_MONTHS(DATE '2004-01-31', 1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "2004-02-29" + } + ] + } + }, + { + "id": "add-months-feb28-keeps-the-day", + "text": "ADD_MONTHS(DATE '2001-02-28', 1)", + "site": "value", + "model": "invoices", + "note": "no month-end preservation: the 28th stays the 28th", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "2001-02-28" + }, + "count": { + "node": "number", + "value": "1" + } + }, + "canonical": "ADD_MONTHS(DATE '2001-02-28', 1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "2001-03-28" + } + ] + } + }, + { + "id": "add-months-mar31-back-one", + "text": "ADD_MONTHS(DATE '2001-03-31', -1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "2001-03-31" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "1" + } + } + }, + "canonical": "ADD_MONTHS(DATE '2001-03-31', -1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "2001-02-28" + } + ] + } + }, + { + "id": "add-months-apr30-forward-one", + "text": "ADD_MONTHS(DATE '2001-04-30', 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "2001-04-30" + }, + "count": { + "node": "number", + "value": "1" + } + }, + "canonical": "ADD_MONTHS(DATE '2001-04-30', 1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "2001-05-30" + } + ] + } + }, + { + "id": "add-months-thirteen", + "text": "ADD_MONTHS(DATE '2000-01-31', 13)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "2000-01-31" + }, + "count": { + "node": "number", + "value": "13" + } + }, + "canonical": "ADD_MONTHS(DATE '2000-01-31', 13)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "2001-02-28" + } + ] + } + }, + { + "id": "add-months-count-column", + "text": "ADD_MONTHS(issue_date, lines)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "column", + "name": "lines" + } + }, + "canonical": "ADD_MONTHS(issue_date, lines)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-01-31", + "lines": "1" + }, + "result": "2026-02-28" + }, + { + "row": { + "issue_date": "2026-01-31", + "lines": "0" + }, + "result": "2026-01-31" + }, + { + "row": { + "issue_date": "2026-01-31", + "lines": "-2" + }, + "result": "2025-11-30" + }, + { + "row": { + "issue_date": "2026-01-31", + "lines": "12" + }, + "result": "2027-01-31" + }, + { + "row": { + "issue_date": "2026-01-31", + "lines": "-13" + }, + "result": "2024-12-31" + }, + { + "row": { + "issue_date": "2026-01-31" + }, + "result": null + }, + { + "row": { + "lines": "1" + }, + "result": null + }, + { + "row": {}, + "result": null + } + ] + } + }, + { + "id": "add-years-feb29", + "text": "ADD_YEARS(issue_date, lines)", + "site": "value", + "model": "invoices", + "note": "ADD_YEARS(d, n) is ADD_MONTHS(d, 12 * n); 2100 is not a leap year, 2400 is", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "year", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "column", + "name": "lines" + } + }, + "canonical": "ADD_YEARS(issue_date, lines)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2000-02-29", + "lines": "-1" + }, + "result": "1999-02-28" + }, + { + "row": { + "issue_date": "2000-02-29", + "lines": "1" + }, + "result": "2001-02-28" + }, + { + "row": { + "issue_date": "2000-02-29", + "lines": "4" + }, + "result": "2004-02-29" + }, + { + "row": { + "issue_date": "2000-02-29", + "lines": "100" + }, + "result": "2100-02-28" + }, + { + "row": { + "issue_date": "2000-02-29", + "lines": "400" + }, + "result": "2400-02-29" + }, + { + "row": { + "issue_date": "2000-02-29", + "lines": "0" + }, + "result": "2000-02-29" + }, + { + "row": { + "issue_date": "2000-02-29", + "lines": null + }, + "result": null + }, + { + "row": { + "issue_date": null, + "lines": "1" + }, + "result": null + } + ] + } + }, + { + "id": "add-days-month-ends", + "text": "ADD_DAYS(issue_date, days_late)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "column", + "name": "days_late" + } + }, + "canonical": "ADD_DAYS(issue_date, days_late)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-01-31", + "days_late": "30" + }, + "result": "2026-03-02" + }, + { + "row": { + "issue_date": "2026-03-01", + "days_late": "-1" + }, + "result": "2026-02-28" + }, + { + "row": { + "issue_date": "2024-03-01", + "days_late": "-1" + }, + "result": "2024-02-29" + }, + { + "row": { + "issue_date": "2026-12-31", + "days_late": "1" + }, + "result": "2027-01-01" + }, + { + "row": { + "issue_date": "2026-11-16", + "days_late": "0" + }, + "result": "2026-11-16" + }, + { + "row": { + "issue_date": "2026-11-16", + "days_late": null + }, + "result": null + }, + { + "row": { + "issue_date": null, + "days_late": "5" + }, + "result": null + } + ] + } + }, + { + "id": "add-days-last-day-in-range", + "text": "ADD_DAYS(DATE '2000-01-01', 2921939)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "date", + "value": "2000-01-01" + }, + "count": { + "node": "number", + "value": "2921939" + } + }, + "canonical": "ADD_DAYS(DATE '2000-01-01', 2921939)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "9999-12-31" + } + ] + } + }, + { + "id": "add-days-one-past-the-last-day", + "text": "ADD_DAYS(DATE '2000-01-01', 2921940)", + "site": "value", + "model": "invoices", + "note": "a result past 9999-12-31 is NULL, never an error", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "date", + "value": "2000-01-01" + }, + "count": { + "node": "number", + "value": "2921940" + } + }, + "canonical": "ADD_DAYS(DATE '2000-01-01', 2921940)", + "type": "date", + "eval": [ + { + "row": {}, + "result": null + } + ] + } + }, + { + "id": "add-days-first-day-in-range", + "text": "ADD_DAYS(DATE '2000-01-01', -730119)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "date", + "value": "2000-01-01" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "730119" + } + } + }, + "canonical": "ADD_DAYS(DATE '2000-01-01', -730119)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "0001-01-01" + } + ] + } + }, + { + "id": "add-days-before-the-first-day", + "text": "ADD_DAYS(DATE '2000-01-01', -730120)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "date", + "value": "2000-01-01" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "730120" + } + } + }, + "canonical": "ADD_DAYS(DATE '2000-01-01', -730120)", + "type": "date", + "eval": [ + { + "row": {}, + "result": null + } + ] + } + }, + { + "id": "add-days-huge-count", + "text": "ADD_DAYS(issue_date, 1000000000000000000000000000000)", + "site": "value", + "model": "invoices", + "note": "any count, however large, gives NULL when the result leaves the range", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "number", + "value": "1000000000000000000000000000000" + } + }, + "canonical": "ADD_DAYS(issue_date, 1000000000000000000000000000000)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-16" + }, + "result": null + } + ] + } + }, + { + "id": "add-days-huge-negative-count", + "text": "ADD_DAYS(issue_date, -1000000000000000000000000000000)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "1000000000000000000000000000000" + } + } + }, + "canonical": "ADD_DAYS(issue_date, -1000000000000000000000000000000)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-16" + }, + "result": null + } + ] + } + }, + { + "id": "add-months-last-month-in-range", + "text": "ADD_MONTHS(DATE '9999-11-30', 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "9999-11-30" + }, + "count": { + "node": "number", + "value": "1" + } + }, + "canonical": "ADD_MONTHS(DATE '9999-11-30', 1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "9999-12-30" + } + ] + } + }, + { + "id": "add-months-past-9999", + "text": "ADD_MONTHS(DATE '9999-12-31', 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "9999-12-31" + }, + "count": { + "node": "number", + "value": "1" + } + }, + "canonical": "ADD_MONTHS(DATE '9999-12-31', 1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": null + } + ] + } + }, + { + "id": "add-months-back-from-9999", + "text": "ADD_MONTHS(DATE '9999-12-31', -1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "9999-12-31" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "1" + } + } + }, + "canonical": "ADD_MONTHS(DATE '9999-12-31', -1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "9999-11-30" + } + ] + } + }, + { + "id": "add-months-before-year-one", + "text": "ADD_MONTHS(DATE '0001-01-01', -1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "0001-01-01" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "1" + } + } + }, + "canonical": "ADD_MONTHS(DATE '0001-01-01', -1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": null + } + ] + } + }, + { + "id": "add-months-first-month-in-range", + "text": "ADD_MONTHS(DATE '0001-12-31', -11)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "date", + "value": "0001-12-31" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "11" + } + } + }, + "canonical": "ADD_MONTHS(DATE '0001-12-31', -11)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "0001-01-31" + } + ] + } + }, + { + "id": "add-months-huge-count", + "text": "ADD_MONTHS(issue_date, 1000000000000000000000000000000)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "month", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "number", + "value": "1000000000000000000000000000000" + } + }, + "canonical": "ADD_MONTHS(issue_date, 1000000000000000000000000000000)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-16" + }, + "result": null + } + ] + } + }, + { + "id": "add-years-past-9999", + "text": "ADD_YEARS(DATE '9999-12-31', 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "year", + "date": { + "node": "date", + "value": "9999-12-31" + }, + "count": { + "node": "number", + "value": "1" + } + }, + "canonical": "ADD_YEARS(DATE '9999-12-31', 1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": null + } + ] + } + }, + { + "id": "add-years-to-9999", + "text": "ADD_YEARS(DATE '0001-01-01', 9998)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "year", + "date": { + "node": "date", + "value": "0001-01-01" + }, + "count": { + "node": "number", + "value": "9998" + } + }, + "canonical": "ADD_YEARS(DATE '0001-01-01', 9998)", + "type": "date", + "eval": [ + { + "row": {}, + "result": "9999-01-01" + } + ] + } + }, + { + "id": "add-years-before-year-one", + "text": "ADD_YEARS(DATE '0001-01-01', -1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "year", + "date": { + "node": "date", + "value": "0001-01-01" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "1" + } + } + }, + "canonical": "ADD_YEARS(DATE '0001-01-01', -1)", + "type": "date", + "eval": [ + { + "row": {}, + "result": null + } + ] + } + }, + { + "id": "add-years-huge-count", + "text": "ADD_YEARS(issue_date, -1000000000000000000000000000000)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "year", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "1000000000000000000000000000000" + } + } + }, + "canonical": "ADD_YEARS(issue_date, -1000000000000000000000000000000)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-16" + }, + "result": null + } + ] + } + }, + { + "id": "add-days-rounded-count", + "text": "ADD_DAYS(issue_date, ROUND(net, 0))", + "site": "value", + "model": "invoices", + "note": "ROUND(x, 0) has scale 0, so it is a count", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "round", + "arg": { + "node": "column", + "name": "net" + }, + "scale": "0", + "mode": "HALF_AWAY_FROM_ZERO" + } + }, + "canonical": "ADD_DAYS(issue_date, ROUND(net, 0))", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-16", + "net": "10.50" + }, + "result": "2026-11-27" + }, + { + "row": { + "issue_date": "2026-11-16", + "net": "10.49" + }, + "result": "2026-11-26" + }, + { + "row": { + "issue_date": "2026-11-16", + "net": null + }, + "result": null + } + ] + } + }, + { + "id": "add-days-counted-by-a-difference", + "text": "ADD_DAYS(issue_date, DAYS_BETWEEN(issue_date, due_date))", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "date_diff", + "unit": "day", + "from": { + "node": "column", + "name": "issue_date" + }, + "to": { + "node": "column", + "name": "due_date" + } + } + }, + "canonical": "ADD_DAYS(issue_date, DAYS_BETWEEN(issue_date, due_date))", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-01", + "due_date": "2026-11-30" + }, + "result": "2026-11-30" + } + ] + } + }, + { + "id": "add-days-count-arithmetic", + "text": "ADD_DAYS(issue_date, lines * 2 - 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "arith", + "op": "-", + "left": { + "node": "arith", + "op": "*", + "left": { + "node": "column", + "name": "lines" + }, + "right": { + "node": "number", + "value": "2" + } + }, + "right": { + "node": "number", + "value": "1" + } + } + }, + "canonical": "ADD_DAYS(issue_date, lines * 2 - 1)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-16", + "lines": "3" + }, + "result": "2026-11-21" + } + ] + } + }, + { + "id": "add-days-negative-literal", + "text": "ADD_DAYS(issue_date, -1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "neg", + "operand": { + "node": "number", + "value": "1" + } + } + }, + "canonical": "ADD_DAYS(issue_date, -1)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-16" + }, + "result": "2026-11-15" + } + ] + } + }, + { + "id": "add-days-nested", + "text": "ADD_DAYS(ADD_MONTHS(issue_date, 1), 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "date_add", + "unit": "month", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "number", + "value": "1" + } + }, + "count": { + "node": "number", + "value": "1" + } + }, + "canonical": "ADD_DAYS(ADD_MONTHS(issue_date, 1), 1)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-01-31" + }, + "result": "2026-03-01" + } + ] + } + }, + { + "id": "add-days-predicate-use", + "text": "due_date = ADD_DAYS(issue_date, 30)", + "site": "predicate", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "compare", + "op": "=", + "left": { + "node": "column", + "name": "due_date" + }, + "right": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "number", + "value": "30" + } + } + }, + "canonical": "due_date = ADD_DAYS(issue_date, 30)", + "type": "bool", + "eval": [ + { + "row": { + "issue_date": "2026-11-01", + "due_date": "2026-12-01" + }, + "result": true + }, + { + "row": { + "issue_date": "2026-11-01", + "due_date": "2026-12-02" + }, + "result": false + }, + { + "row": { + "issue_date": "2026-11-01" + }, + "result": false + }, + { + "row": { + "due_date": "2026-12-01" + }, + "result": false + } + ] + } + }, + { + "id": "add-days-derive-date", + "text": "ADD_DAYS(issue_date, 30)", + "site": "value", + "model": "invoices", + "derive": "due_date", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "number", + "value": "30" + } + }, + "canonical": "ADD_DAYS(issue_date, 30)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-01" + }, + "result": "2026-12-01" + }, + { + "row": { + "issue_date": null + }, + "result": null + } + ] + } + }, + { + "id": "add-months-then-months-between", + "text": "MONTHS_BETWEEN(issue_date, ADD_MONTHS(issue_date, lines))", + "site": "value", + "model": "invoices", + "note": "the inverse holds whenever the result is in range", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_diff", + "unit": "month", + "from": { + "node": "column", + "name": "issue_date" + }, + "to": { + "node": "date_add", + "unit": "month", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "column", + "name": "lines" + } + } + }, + "canonical": "MONTHS_BETWEEN(issue_date, ADD_MONTHS(issue_date, lines))", + "type": "number", + "scale": "0", + "eval": [ + { + "row": { + "issue_date": "2026-01-31", + "lines": "1" + }, + "result": "1" + }, + { + "row": { + "issue_date": "2026-01-31", + "lines": "-13" + }, + "result": "-13" + }, + { + "row": { + "issue_date": "2000-02-29", + "lines": "12" + }, + "result": "12" + }, + { + "row": { + "issue_date": "2026-03-31", + "lines": "-1" + }, + "result": "-1" + }, + { + "row": { + "issue_date": "9999-12-31", + "lines": "1" + }, + "result": null + } + ] + } + }, + { + "id": "add-days-lower-case", + "text": "add_days(issue_date, 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "accepted", + "tree": { + "node": "date_add", + "unit": "day", + "date": { + "node": "column", + "name": "issue_date" + }, + "count": { + "node": "number", + "value": "1" + } + }, + "canonical": "ADD_DAYS(issue_date, 1)", + "type": "date", + "eval": [ + { + "row": { + "issue_date": "2026-11-16" + }, + "result": "2026-11-17" + } + ] + } + }, + { + "id": "add-days-as-a-column-name", + "text": "add_days > 1", + "site": "predicate", + "model": "invoices", + "note": "a function name is not a keyword: without a '(' it is a column", + "expect": { + "verdict": "accepted", + "tree": { + "node": "compare", + "op": ">", + "left": { + "node": "column", + "name": "add_days" + }, + "right": { + "node": "number", + "value": "1" + } + }, + "canonical": "add_days > 1", + "type": "bool", + "eval": [ + { + "row": { + "add_days": "2" + }, + "result": true + }, + { + "row": { + "add_days": "1" + }, + "result": false + } + ] + } + }, + { + "id": "add-days-count-fraction-literal", + "text": "ADD_DAYS(issue_date, 1.5)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-days-count-scale-two", + "text": "ADD_DAYS(issue_date, net)", + "site": "value", + "model": "invoices", + "note": "net is decimal(12,2): a count has scale 0", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-months-count-scale-two", + "text": "ADD_MONTHS(issue_date, net)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-days-count-float", + "text": "ADD_DAYS(issue_date, fx_rate)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-years-count-float", + "text": "ADD_YEARS(issue_date, fx_rate)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-days-count-unscaled-decimal", + "text": "ADD_DAYS(issue_date, amount_any)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-days-count-quotient", + "text": "ADD_DAYS(issue_date, lines / 2)", + "site": "value", + "model": "invoices", + "note": "a division has no static scale", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-days-count-string", + "text": "ADD_DAYS(issue_date, invoice_id)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-days-date-is-timestamp", + "text": "ADD_DAYS(issued_at, 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-days-date-is-string", + "text": "ADD_DAYS(invoice_id, 1)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-days-null-date", + "text": "ADD_DAYS(NULL, 1)", + "site": "value", + "model": "invoices", + "note": "unlike the date differences, the NULL literal takes no type from the other argument", + "expect": { + "verdict": "refused", + "reason": "null-literal" + } + }, + { + "id": "add-days-null-count", + "text": "ADD_DAYS(issue_date, NULL)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "null-literal" + } + }, + { + "id": "add-days-one-argument", + "text": "ADD_DAYS(issue_date)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "syntax" + } + }, + { + "id": "add-days-three-arguments", + "text": "ADD_DAYS(issue_date, 1, 2)", + "site": "value", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "syntax" + } + }, + { + "id": "add-days-as-a-predicate", + "text": "ADD_DAYS(issue_date, 1)", + "site": "predicate", + "model": "invoices", + "expect": { + "verdict": "refused", + "reason": "type-mismatch" + } + }, + { + "id": "add-days-derived-into-int", + "text": "ADD_DAYS(issue_date, 1)", + "site": "value", + "model": "invoices", + "derive": "lines", + "expect": { + "verdict": "refused", + "reason": "derive-type" + } + }, { "id": "date-literal-compare", "text": "issue_date < DATE '2026-11-16'", diff --git a/fluid_build/profiler/_stats.py b/fluid_build/profiler/_stats.py index f46b78a1..6c60137e 100644 --- a/fluid_build/profiler/_stats.py +++ b/fluid_build/profiler/_stats.py @@ -37,6 +37,7 @@ from importlib import import_module from types import SimpleNamespace from typing import ( + TYPE_CHECKING, Any, Callable, Dict, @@ -50,6 +51,9 @@ runtime_checkable, ) +if TYPE_CHECKING: + from fluid_build.profiler.defaults import DeclaredRange + __all__ = [ "BlockRefused", "BundleTextRefused", @@ -168,12 +172,21 @@ class DeclaredBlock: """What the fidelity block asks of a profile, read by the registry's own parser. ``left_out`` names each declared property that is not answered, with the - reason. The parsed block stays private to this seam, which measures with it. + reason. ``every_declared_group`` is set for the block the profiler declares + from the contract (:mod:`fluid_build.profiler.defaults`): a grouped statistic + then lists every level its ``groupBy`` declares as a group, one no record holds + included, so the groups a document lists never depend on the records. That block's + ``declared_ranges`` are the ranges its histograms cover, as the contract's + validationRules declare them: each value is checked against its range exactly before + it is counted, since the bins are rounded outward to the grid. The parsed block + stays private to this seam, which measures with it. """ fractional_bits: int statistics: Tuple[DeclaredStatistic, ...] left_out: Tuple[Tuple[str, str], ...] = () + every_declared_group: bool = False + declared_ranges: Mapping[str, "DeclaredRange"] = field(default_factory=dict, compare=False) _block: Any = field(default=None, repr=False, compare=False) @property @@ -363,6 +376,13 @@ def read_block( :class:`StatisticsUnavailable` when the registry has no grammar. """ + def enum_levels(self, declared_type: str) -> Tuple[str, ...]: + """The levels a FLUID ``enum(...)`` column type declares, as the registry reads them. + + ``()`` for any other type. Raises :class:`BlockRefused` for a level list the + registry's grammar refuses (``FLX-3001``). + """ + def measure( self, block: DeclaredBlock, @@ -610,6 +630,14 @@ def _fidelity(self) -> Any: ) return self._grammar + def enum_levels(self, declared_type: str) -> Tuple[str, ...]: + grammar = self._fidelity() + try: + levels: Tuple[str, ...] = grammar.fidelity.declared_enum_values(declared_type) + except self._refused as error: + raise BlockRefused(error.code.code, error.detail or error.code.title) from None + return levels + def read_block( self, raw_block: Any, contract_document: Mapping[str, Any], expose_id: str ) -> DeclaredBlock: @@ -723,6 +751,14 @@ def measure( # the engine's synthetic rows hold declared levels alone, so a check loses # no group it could compare. keys = grammar.reading.group_keys(rows, by, declared_only=True) + if block.every_declared_group and by is not None: + # A declared level no record holds is a group of none, so the groups listed + # are the declared levels whatever the records hold; the check for counts + # at edges withholds such a group with its siblings. + present = {key for key, _ in keys} + keys = sorted( + [*keys, *((level, []) for level in by.support_values if level not in present)] + ) groups = tuple( self._measured(grammar, parsed, declaration, statistic, rows, key, indices) for key, indices in keys diff --git a/fluid_build/profiler/defaults.py b/fluid_build/profiler/defaults.py new file mode 100644 index 00000000..f1d8a546 --- /dev/null +++ b/fluid_build/profiler/defaults.py @@ -0,0 +1,447 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""The statistics a profile carries without ``--bundle``, declared from the contract alone. + +Without a Simulation to answer, the profiler declares, from the contract and never +from the records, the statistics ``fluxsim init`` reads, and writes them as a fidelity +block (SEAM-FIDELITY-SPEC §2). The registry's own grammar parses that block exactly as +it parses a Simulation's under ``--bundle``, so there is no second parser, and every +statistic passes the same output checks: ``check_edge_counts`` for the counts at a +column's edges (FLUX ADR 0012) and the ACRO rules for the rest. + +- **A column typed** ``enum(...)``: ``levelHistogram/1`` over every level the type + declares. A level no record holds is a cell of none, which the check withholds as + ``C``, so the levels listed never depend on the records. +- **A number whose** ``validationRules`` **declare both ends of a range** inside its + type: ``marginalHistogram/1`` over about ten bins of one width, exact at both ends (an + integer range ends half a unit past its top, a whole unit on a grid of whole steps, so + each value has a bin of its own). A side at or past the type's own bound declares + nothing the type does not, and a range of one side is not closed: ten bins over the rest + of the type would hold every record in one of them, so such a number takes the ladder. +- **Any other number of a bounded type, a date or a timestamp**: ``quantile/2`` at the + lower quartile, the median and the upper quartile, on a ladder fixed by the type: 0, + ±1, ±2, ±5, ±10, ±20, ±50 ... within the type's bounds for a number, one bin per year + from 1900 to 2100 for a date or a timestamp. No histogram is declared on a ladder: its + end bins hold no record, so the check can tie no published cell to an end of the range, + and it withholds a quantile beside the histogram first. Measured on ages, amounts, ids + and counts, a histogram on the ladder published nothing, and the quantiles beside it + nothing. +- **A grouping the contract declares**, a categorical ``semantics.dimensions`` entry on + an ``enum`` column: each other ``enum`` column's level histogram per level of the + grouping. Every declared level is a group, one that no record holds included, so the + groups listed never depend on the records either; such a group withholds its siblings. + One column's groupings hold at most :data:`MOST_GROUPED_CELLS` cells together: one past + that is left out, and named. + +A column gets a histogram or quantiles, never both on one set of bins: checked together, +the two let a reader who knows the check's choices tell worlds apart, as +``tests/profiler/test_disclosure_oracle.py`` found. For the same reason the runner +withholds a column's ``distinctCount`` beside a level histogram, and a number's ``sum``, +``min``, ``max``, ``quantiles`` and ``distinctCount`` beside a histogram over its declared +range, where it forms no ``mean/1``. + +A number with neither a range of two ends nor a bounded type (a ``decimal`` without +``(p,s)``) gets none, and the owner is told. A record outside what the contract +declares, a level its type does not list or a value outside its type or the range, +refuses the profile, as one outside a Simulation's block does. The bins of a range are +rounded outward to the grid, which can be coarser than the column's own scale, so each +value is checked against the range itself first (:class:`DeclaredRange`). + +The tolerances and the ``records`` and ``budget`` sections are fields the grammar +requires of any block; a profile reads none of them. +""" + +from __future__ import annotations + +import math +import re +from dataclasses import dataclass, field +from datetime import datetime, timezone +from fractions import Fraction +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple + +__all__ = [ + "DeclaredRange", + "DefaultPlan", + "MOST_GROUPED_CELLS", + "QUANTILE_LEVELS", + "default_plan", + "declared_range", + "range_bins", + "number_ladder", +] + +#: The quantile levels declared on every number, date and timestamp: the quartiles and +#: the median, as the ``quantiles`` metric is measured. +QUANTILE_LEVELS: Tuple[str, ...] = ("0.25", "0.5", "0.75") + +#: About how many bins a declared range is cut into. +RANGE_BINS = 10 + +#: The bounds an integer column is read within: FLUX writes every integer as int64. +_INT64 = (-(2**63), 2**63 - 1) + +#: The widest number a bin edge spells: 10^38, the widest SQL ``DECIMAL``. A wider type or +#: declared bound gets no statistic: its edges could pass the 1,000 digits flux-stats +#: writes out, which would refuse the whole profile for one column. +_WIDEST = Fraction(10) ** 38 + +#: The bounds a date or a timestamp is read within, and the years its ladder steps by. +_FIRST_INSTANT = "0001-01-01T00:00:00Z" +_LAST_INSTANT = "9999-12-31T23:59:59.999Z" +_LADDER_YEARS = range(1900, 2101) + +#: The smallest sub-unit edge a decimal's ladder takes: 2^-10, and never below the grid. +_FINEST_SUB_UNIT = 10 + +#: The most cells one column's grouped level histograms hold together, every declared level +#: of each grouping a group. Checking them together costs about the cube of the levels, so +#: a grouping past this is left out, and named: the contract alone decides, never a record. +MOST_GROUPED_CELLS = 10_000 + +_DISTANCE = {"type": "distance", "measure": "totalVariation", "max": "1"} +_ABSOLUTE = {"type": "absolute", "value": "1"} + + +@dataclass(frozen=True) +class DeclaredRange: + """The range a number's ``validationRules`` declare, read exactly as they state it. + + The bins of a histogram over it are rounded outward to the grid, which can be coarser + than the column's own scale, so the runner checks every value against this range + before the value is counted: :meth:`holds` is exact, and a strict bound (``>``, + ``<``) is not reached. + """ + + low: Fraction + high: Fraction + low_strict: bool = False + high_strict: bool = False + + def holds(self, value: Fraction) -> bool: + """Whether *value* lies in the range, each bound as strict as its rule.""" + above = value > self.low if self.low_strict else value >= self.low + below = value < self.high if self.high_strict else value <= self.high + return above and below + + +@dataclass(frozen=True) +class DefaultPlan: + """The block a profile answers without ``--bundle``, and the columns it cannot cover. + + ``block`` is ``None`` when the contract declares nothing to answer. ``uncovered`` + names each number, date or timestamp column with no statistic, with why. ``ranges`` + holds the declared range of each column the block gives a histogram over one. + """ + + block: Optional[Dict[str, Any]] + uncovered: Tuple[Tuple[str, str], ...] = () + ranges: Mapping[str, DeclaredRange] = field(default_factory=dict) + + +def declared_range(column: Mapping[str, Any]) -> Tuple[Optional[Fraction], Optional[Fraction]]: + """The exact bounds a column's ``range`` validationRules declare: ``(low, high)``. + + The grammar is the one ``fluid test`` reads (``quality_engine.parse_range``), read + without rounding (``parse_range_exact``). Several rules hold together, so the + tightest bound of each side is kept. A rule the grammar does not read declares + nothing, and so does a ``validationRules`` that is not a list. + """ + (low, _), (high, _) = _declared_sides(column) + return low, high + + +_Side = Tuple[Optional[Fraction], bool] + + +def _declared_sides(column: Mapping[str, Any]) -> Tuple[_Side, _Side]: + """:func:`declared_range`'s two bounds, each with whether its rule is strict.""" + from fluid_build.providers.quality_engine import parse_range_exact + + low: _Side = (None, False) + high: _Side = (None, False) + rules = column.get("validationRules") + for rule in rules if isinstance(rules, list) else (): + if not isinstance(rule, Mapping) or str(rule.get("type") or "").lower() != "range": + continue + for operator, value in parse_range_exact(rule.get("constraint")): + strict = operator in (">", "<") + if operator in (">", ">="): + if low[0] is None or value > low[0] or (value == low[0] and strict): + low = (value, strict) + elif high[0] is None or value < high[0] or (value == high[0] and strict): + high = (value, strict) + return low, high + + +def _type_bounds(declared: Any) -> Tuple[Optional[Tuple[Fraction, Fraction]], bool]: + """The bounds a number type holds every value within, and whether its scale is above 0. + + An integer type is read as int64. ``decimal(p,s)`` holds less than ``10^(p-s)`` either + way. A decimal without ``(p,s)`` has no bound. The decimal is read as FLUID SQL/1 + reads one (``fluid_sql._types.column_type``), so ``decimal(0012, 02)`` is ``(12, 2)``. + """ + from fluid_build.fluid_sql._types import NUMBER, column_type + + read = column_type(declared) + if read is None or read.type.kind != NUMBER or read.is_float or read.type.scale is None: + return None, False + scale = read.type.scale + if read.precision is None: # an integer type + return (Fraction(_INT64[0]), Fraction(_INT64[1])), False + bound = Fraction(10) ** (read.precision - scale) + if bound > _WIDEST: # wider than any edge is spelled: no bound a bin can use + return None, False + return (-bound, bound), scale > 0 + + +def _scale(declared: Any) -> Optional[int]: + """A number type's static scale as FLUID SQL/1 reads it: 0 for an integer type.""" + from fluid_build.fluid_sql._types import column_type + + read = column_type(declared) + return None if read is None or read.is_float else read.type.scale + + +def number_ladder( + low: Fraction, high: Fraction, *, sub_unit: bool, fractional_bits: int +) -> Tuple[Fraction, ...]: + """The edges 0, ±1, ±2, ±5, ±10, ... strictly inside ``(low, high)``, and both bounds. + + With *sub_unit*, also ±1/2, ±1/4 ... down to 2^-10 or the grid's step, whichever is + coarser, so that a column of rates or shares has bins below 1. Every edge is a grid + point: an integer, or a power of two the grid holds. + """ + edges = {low, high} + if low < 0 < high: + edges.add(Fraction(0)) + magnitude = Fraction(1) + while magnitude < max(-low, high): + for step in (1, 2, 5): + for edge in (magnitude * step, -magnitude * step): + if low < edge < high: + edges.add(edge) + magnitude *= 10 + if sub_unit: + for power in range(1, min(_FINEST_SUB_UNIT, fractional_bits) + 1): + for edge in (Fraction(1, 2**power), -Fraction(1, 2**power)): + if low < edge < high: + edges.add(edge) + return tuple(sorted(edges)) + + +def range_bins( + low: Fraction, high: Fraction, fractional_bits: int, *, integral: bool = False +) -> Tuple[Fraction, ...]: + """About :data:`RANGE_BINS` bins over the declared range ``[low, high]``, and no wider. + + The edges are the range's two ends and the multiples of one width strictly between + them: the narrowest of 2^-k (on the grid), 1, 2, 5, 10, 20, 50 ... that cuts the range + into at most :data:`RANGE_BINS` bins. An *integral* column (an integer type, or a + decimal of scale 0) takes whole widths only, and its range ends half a unit past + ``high`` (a whole unit on a grid of whole steps, which holds no half), so the last + value is not alone with the one below it in the last bin, which is closed at the top: + ``between 1 and 5`` gets one bin per value. The ends are rounded outward to the grid. + The grid can be coarser than the column's own scale, so an end bin can reach past a + bound: the runner checks each value against the range itself before it is counted + (:class:`DeclaredRange`). + """ + step = Fraction(1, 2**fractional_bits) + first = math.floor(low / step) * step + top = high + (Fraction(1, 2) if fractional_bits >= 1 else 1) if integral else high + last = math.ceil(top / step) * step + span = high - low + widths: List[Fraction] = ( + [] if integral else [Fraction(1, 2**power) for power in range(fractional_bits, 0, -1)] + ) + magnitude = 1 + while not widths or widths[-1] * RANGE_BINS < span: + widths.extend(Fraction(magnitude * scale) for scale in (1, 2, 5)) + magnitude *= 10 + width = next(each for each in widths if each * RANGE_BINS >= span) + inner = [] + edge = (math.floor(first / width) + 1) * width + while edge < last: + inner.append(edge) + edge += width + return (first, *inner, last) + + +def _instant(year: int) -> str: + return datetime(year, 1, 1, tzinfo=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _decimal(value: Fraction) -> str: + """An exact edge as a decimal string: every edge here is a terminating decimal.""" + if value.denominator == 1: + return str(value.numerator) + places = 0 + while (value * 10**places).denominator != 1: + places += 1 + scaled = value * 10**places + digits = str(abs(scaled.numerator)).rjust(places + 1, "0") + sign = "-" if scaled < 0 else "" + return f"{sign}{digits[:-places]}.{digits[-places:]}" + + +def _dimension_columns(expose: Mapping[str, Any], names: Sequence[str]) -> List[str]: + """The columns a categorical ``semantics.dimensions`` entry groups by, in order, once. + + A dimension names its column by ``expr`` or, without one, by its ``name``; an + ``expr`` that is not a bare column name is a computed grouping, not a column's. + """ + semantics = expose.get("semantics") + dimensions = semantics.get("dimensions") if isinstance(semantics, Mapping) else None + found: List[str] = [] + for dimension in dimensions if isinstance(dimensions, list) else (): + if not isinstance(dimension, Mapping) or dimension.get("type") != "categorical": + continue + column = dimension.get("expr") or dimension.get("name") + if isinstance(column, str) and column in names and column not in found: + found.append(column) + return found + + +def default_plan( + expose: Mapping[str, Any], + columns: Sequence[Any], + enum_levels: Callable[[str], Sequence[str]], + fractional_bits: int, +) -> DefaultPlan: + """The fidelity block the contract's declarations give *expose*, on the profile's grid. + + *columns* are the expose's profiled columns (``runner.ColumnPlan``: name and domain) + and *enum_levels* reads the levels a FLUID ``enum(...)`` type declares, as the + registry reads them (:meth:`StatisticsBackend.enum_levels`). + """ + contract = expose.get("contract") + entries = contract.get("schema") if isinstance(contract, Mapping) else None + schema = { + entry["name"]: entry + for entry in (entries if isinstance(entries, list) else ()) + if isinstance(entry, Mapping) and isinstance(entry.get("name"), str) + } + declared: Dict[str, Any] = {} + ranges: Dict[str, DeclaredRange] = {} + properties: List[Dict[str, Any]] = [] + uncovered: List[Tuple[str, str]] = [] + levels: Dict[str, int] = {} + for column in columns: + entry = schema.get(column.name, {}) + declared_type = entry.get("type") + if column.domain == "category": + try: + found = _levels(enum_levels, declared_type) + except ValueError as refused: # the registry's FLX-3001: a list it cannot read + uncovered.append((column.name, f"its enum(...) type does not read ({refused})")) + continue + if found: + levels[column.name] = found + declared[column.name] = {"support": "declared", "maxCardinality": found} + continue + if column.domain == "timestamp": + declared[column.name] = {"range": {"min": _FIRST_INSTANT, "max": _LAST_INSTANT}} + ladder = [_FIRST_INSTANT, *(_instant(year) for year in _LADDER_YEARS), _LAST_INSTANT] + properties.extend(_quantiles(column.name, ladder)) + continue + bounds, sub_unit = _type_bounds(declared_type) + (low, low_strict), (high, high_strict) = _declared_sides(entry) + if bounds is not None: + # A side at or past the type's own bound narrows nothing: bins over the rest + # of the type would hold every record in one of them, and publish none. + low = low if low is not None and low > bounds[0] else None + high = high if high is not None and high < bounds[1] else None + if low is not None and high is not None and max(-low, high) > _WIDEST: + uncovered.append((column.name, "its validationRules range is wider than 10^38")) + elif low is not None and high is not None and low < high: + integral = _scale(declared_type) == 0 + edges = [ + _decimal(edge) for edge in range_bins(low, high, fractional_bits, integral=integral) + ] + declared[column.name] = {"range": {"min": edges[0], "max": edges[-1]}} + ranges[column.name] = DeclaredRange(low, high, low_strict, high_strict) + properties.append( + { + "property": "marginalHistogram", + "column": column.name, + "bins": edges, + "tolerance": dict(_DISTANCE), + } + ) + elif low is not None and high is not None: + uncovered.append((column.name, "its validationRules range is empty or inverted")) + elif bounds is not None: # no range, or one of one side: the type's own ladder + steps = number_ladder( + bounds[0], bounds[1], sub_unit=sub_unit, fractional_bits=fractional_bits + ) + edges = [_decimal(edge) for edge in steps] + declared[column.name] = {"range": {"min": edges[0], "max": edges[-1]}} + properties.extend(_quantiles(column.name, edges)) + elif low is not None or high is not None: + uncovered.append( + (column.name, "its validationRules range bounds one side, and its type no side") + ) + else: + uncovered.append( + (column.name, "neither its type nor a validationRules range bounds it within 10^38") + ) + for name in levels: + properties.append( + {"property": "levelHistogram", "column": name, "tolerance": dict(_DISTANCE)} + ) + grouped_cells: Dict[str, int] = {} + for by in _dimension_columns(expose, list(levels)): + for name in levels: + if name == by: + continue + cells = grouped_cells.get(name, 0) + levels[by] * levels[name] + if cells > MOST_GROUPED_CELLS: + uncovered.append( + ( + name, + f"its level histogram per level of {by!r}, past the " + f"{MOST_GROUPED_CELLS:,} cells one column's groupings may hold", + ) + ) + continue + grouped_cells[name] = cells + properties.append( + { + "property": "levelHistogram", + "column": name, + "groupBy": by, + "tolerance": dict(_DISTANCE), + } + ) + if not properties: + return DefaultPlan(block=None, uncovered=tuple(uncovered)) + block = { + "encoding": {"fractionalBits": fractional_bits}, + "records": {"min": 1, "max": 1}, + "budget": {"maxChecksPerState": 1}, + "columns": declared, + "properties": properties, + } + return DefaultPlan(block=block, uncovered=tuple(uncovered), ranges=ranges) + + +def _levels(enum_levels: Callable[[str], Sequence[str]], declared_type: Any) -> int: + """How many levels a column's ``enum(...)`` type declares; 0 for any other type.""" + text = str(declared_type or "") + if not re.match(r"\s*enum\s*\(", text, re.IGNORECASE): + return 0 + return len(enum_levels(text)) + + +def _quantiles(column: str, edges: Sequence[str]) -> List[Dict[str, Any]]: + return [ + { + "property": "quantile", + "column": column, + "level": level, + "bins": list(edges), + "tolerance": dict(_ABSOLUTE), + } + for level in QUANTILE_LEVELS + ] diff --git a/fluid_build/profiler/runner.py b/fluid_build/profiler/runner.py index 91d7aed6..09609398 100644 --- a/fluid_build/profiler/runner.py +++ b/fluid_build/profiler/runner.py @@ -35,13 +35,14 @@ import re from collections import Counter -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from datetime import date from fractions import Fraction from pathlib import Path from typing import Any, Dict, FrozenSet, List, Mapping, Optional, Sequence, Tuple from fluid_build.policy._common import iter_exposes -from fluid_build.profiler import disclosure, document, sources +from fluid_build.profiler import defaults, disclosure, document, sources from fluid_build.profiler._stats import ( WITHHELD_ROUNDING, BlockRefused, @@ -63,11 +64,15 @@ "DEFAULT_QUANTILE_LEVELS", "ColumnPlan", "ExposePlan", + "ExposeSource", "ProfileError", "ProfileResult", + "TableResult", "column_domain", + "expose_ids", "plan_expose", "profile_expose", + "profile_exposes", "resolve_expose_id", ] @@ -151,6 +156,8 @@ def __init__(self, event: str, message: str, context: Optional[Dict[str, str]] = class ColumnPlan: name: str domain: str + #: The contract types the column ``date``: a ``YYYY-MM-DD`` text in it is that day. + date: bool = False @dataclass(frozen=True) @@ -168,12 +175,42 @@ class ExposePlan: PartlyNamedStatistic = Tuple[str, str, Optional[str], Optional[str], int, int] +@dataclass(frozen=True) +class ExposeSource: + """One expose a profile reads, and where its records live (``--source``, ``--table``).""" + + expose_id: str + source: str + table: Optional[str] = None + + +@dataclass(frozen=True) +class TableResult: + """What the profile says about one expose, for the owner's summary.""" + + plan: ExposePlan + #: Each published fidelity statistic (:data:`NamedStatistic`). + statistics: Tuple[NamedStatistic, ...] = () + #: Each fidelity statistic written with every group withheld: a run that checks + #: it scores it censored. + withheld_statistics: Tuple[NamedStatistic, ...] = () + #: Each fidelity statistic with some groups published and some withheld: a run + #: that checks it scores only the published groups. + partly_withheld_statistics: Tuple[PartlyNamedStatistic, ...] = () + #: Each declared property the document does not answer, with the reason. + left_out: Tuple[Tuple[str, str], ...] = () + #: Each number, date or timestamp column the contract gives no default statistic, + #: with why (:mod:`fluid_build.profiler.defaults`). + uncovered: Tuple[Tuple[str, str], ...] = () + + @dataclass(frozen=True) class ProfileResult: document: Dict[str, Any] canonical: bytes digest: str - plan: ExposePlan + #: One result per expose, in the order they were profiled. + tables: Tuple[TableResult, ...] published: int #: Withheld values per ``CL_CONF_STATUS`` code of the check that withheld them: a #: histogram's or a quantile's withheld groups are written ``C`` whatever the check, and @@ -184,22 +221,44 @@ class ProfileResult: withheld_cells: Mapping[str, int] = field(default_factory=dict) #: The grid the document declares and its on-grid statistics were formed on. fractional_bits: int = DEFAULT_FRACTIONAL_BITS - #: Each published fidelity statistic (:data:`NamedStatistic`). - statistics: Tuple[NamedStatistic, ...] = () - #: Each fidelity statistic written with every group withheld: a run that checks - #: it scores it censored. - withheld_statistics: Tuple[NamedStatistic, ...] = () - #: Each fidelity statistic with some groups published and some withheld: a run - #: that checks it scores only the published groups. - partly_withheld_statistics: Tuple[PartlyNamedStatistic, ...] = () - #: Each declared property the document does not answer, with the reason. - left_out: Tuple[Tuple[str, str], ...] = () + + def _only(self) -> TableResult: + if len(self.tables) != 1: + raise ValueError("a profile of several exposes has one result per expose: read .tables") + return self.tables[0] + + # The one expose of a profile of one expose, as these were before several could be read. + + @property + def plan(self) -> ExposePlan: + return self._only().plan + + @property + def statistics(self) -> Tuple[NamedStatistic, ...]: + return self._only().statistics + + @property + def withheld_statistics(self) -> Tuple[NamedStatistic, ...]: + return self._only().withheld_statistics + + @property + def partly_withheld_statistics(self) -> Tuple[PartlyNamedStatistic, ...]: + return self._only().partly_withheld_statistics + + @property + def left_out(self) -> Tuple[Tuple[str, str], ...]: + return self._only().left_out + + +def _base_type(type_name: Any) -> str: + """A FLUID column type in lower case, without its parameters: ``decimal(10,2)`` is ``decimal``.""" + text = re.sub(r"\s+", " ", str(type_name or "").strip().lower()) + return re.sub(r"\s*\(.*\)$", "", text) def column_domain(type_name: Any) -> Tuple[Optional[str], Optional[str]]: """The profile domain a FLUID column type is read in, or ``None`` and the reason.""" - text = re.sub(r"\s+", " ", str(type_name or "").strip().lower()) - base = re.sub(r"\s*\(.*\)$", "", text) + base = _base_type(type_name) if base in _CATEGORY_TYPES: return CATEGORY, None if base in _NUMBER_TYPES: @@ -211,9 +270,14 @@ def column_domain(type_name: Any) -> Tuple[Optional[str], Optional[str]]: return None, "the type is not one fluid profile reads" +def expose_ids(contract: Mapping[str, Any]) -> List[str]: + """The contract's exposeIds, in the order it declares them.""" + return [str(item["exposeId"]) for item in iter_exposes(dict(contract)) if item.get("exposeId")] + + def resolve_expose_id(contract: Mapping[str, Any], requested: Optional[str]) -> str: """The expose to profile: *requested*, or the contract's only expose.""" - ids = [str(item["exposeId"]) for item in iter_exposes(dict(contract)) if item.get("exposeId")] + ids = expose_ids(contract) if requested: if requested in ids: return requested @@ -265,7 +329,8 @@ def plan_expose(contract: Mapping[str, Any], expose_id: str) -> ExposePlan: if domain is None: skipped.append((name, reason or "")) else: - columns.append(ColumnPlan(name=name, domain=domain)) + is_date = _base_type(entry.get("type")) == "date" + columns.append(ColumnPlan(name=name, domain=domain, date=is_date)) return ExposePlan(expose_id=expose_id, columns=tuple(columns), skipped=tuple(skipped)) @@ -282,11 +347,13 @@ def _column( tally: "Counter[str]", fractional_bits: int, declares_mean: bool = False, + withhold: FrozenSet[str] = frozenset(), ) -> Dict[str, Any]: """One column of the table: its domain, then its checked metrics and statistics. *declares_mean* says the fidelity block declares an ungrouped ``mean`` of this column: that statistic replaces the fixed ``mean/1``, which is not formed. + *withhold* names the metrics withheld whatever their value (:func:`_counted_levels`). """ read = [ backend.read_value(column.domain, value, column=column.name, row_index=index) @@ -294,7 +361,7 @@ def _column( ] entry: Dict[str, Any] = { "domain": column.domain, - "metrics": _column_metrics(column, read, row_count, backend, rule, tally), + "metrics": _column_metrics(column, read, row_count, backend, rule, tally, withhold), } statistics = ( [] @@ -313,8 +380,13 @@ def _column_metrics( backend: StatisticsBackend, rule: disclosure.DisclosureRule, tally: "Counter[str]", + withhold: FrozenSet[str] = frozenset(), ) -> Dict[str, Any]: - """The checked ``metrics`` object of one column, from its values as read.""" + """The checked ``metrics`` object of one column, from its values as read. + + A metric *withhold* names is withheld under ``D`` whatever its value and its own + check, so its status says nothing about the records either. + """ present = [value for value in read if value is not None] holders = Counter(present) counted = disclosure.check_count(len(present), rule) @@ -341,6 +413,8 @@ def _column_metrics( verdict, shown = counted, value if counted.published: verdict, shown = _check_described(name, value, present, holders, rule) + if name in withhold: + verdict = disclosure.Verdict(disclosure.SECONDARY, "counts what a histogram withholds") entry: Dict[str, Any] = { "definition": metric.identifier, "status": verdict.status, @@ -459,13 +533,195 @@ def profile_expose( ) -> ProfileResult: """Profile *expose_id* of *contract* from the records *source* names. + :func:`profile_exposes` of this one expose: *bundle* is the fidelity block it answers. + """ + return profile_exposes( + contract, + [ExposeSource(expose_id, source, table)], + backend=backend, + cwd=cwd, + rule=rule, + producer=producer, + fractional_bits=fractional_bits, + bundles=None if bundle is None else {expose_id: bundle}, + ) + + +def profile_exposes( + contract: Mapping[str, Any], + exposes: Sequence[ExposeSource], + *, + backend: StatisticsBackend, + cwd: Optional[Path] = None, + rule: disclosure.DisclosureRule = disclosure.ACRO_DEFAULT, + producer: Optional[str] = None, + fractional_bits: Optional[int] = None, + bundles: Optional[Mapping[str, BundleBlock]] = None, +) -> ProfileResult: + """Profile each of *exposes* of *contract*, each from its own source, into one document. + *fractional_bits* is the grid ``mean/1`` is formed on (16 unless given), checked before any record is read; the fidelity block must declare the same - (PROFILE-SPEC §7). With a *bundle*, the profile also answers its fidelity - block, on the block's grid (:func:`_declared_block`). + (PROFILE-SPEC §7). An expose *bundles* holds a block for also answers that + block, on the block's grid (:func:`_declared_block`), and the blocks of one + document declare one grid. Every expose's plan and block is checked before any + record is read, and each expose's records are read once. """ if fractional_bits is not None: document.check_fractional_bits(fractional_bits) + if not exposes: + raise ProfileError( + "profile_expose_required", "no expose to profile", {"next": "name one with --expose"} + ) + _refuse_repeated(exposes) + plans = [_plan_with_columns(contract, each.expose_id) for each in exposes] + blocks = bundles or {} + declared: List[Optional[DeclaredBlock]] = [] + grids: Dict[str, int] = {} + for plan in plans: + found, grid = _declared_block( + blocks.get(plan.expose_id), contract, plan.expose_id, plan, backend, fractional_bits + ) + declared.append(found) + if found is not None: + grids[plan.expose_id] = grid + fractional_bits = _one_grid(grids, blocks, fractional_bits) + uncovered: List[Tuple[Tuple[str, str], ...]] = [() for _ in plans] + for position, plan in enumerate(plans): + if plan.expose_id not in blocks: + declared[position], uncovered[position] = _default_block( + contract, plan, backend, fractional_bits + ) + reads = [ + _dates_read( + plan, + sources.read_columns( + each.source, [column.name for column in plan.columns], table=each.table, cwd=cwd + ), + ) + for each, plan in zip(exposes, plans, strict=True) + ] + tallies: Tuple["Counter[str]", "Counter[str]"] = (Counter(), Counter()) + tables_doc: Dict[str, Dict[str, Any]] = {} + results: List[TableResult] = [] + for plan, read, found, missing in zip(plans, reads, declared, uncovered, strict=True): + table_doc, result = _profile_table( + plan, + read, + found, + backend, + rule, + tallies, + fractional_bits, + ) + tables_doc[plan.expose_id] = table_doc + results.append(replace(result, uncovered=missing)) + if producer is None: + from fluid_build import __version__ + + producer = document.producer_uri(__version__) + built = document.build_document( + producer=producer, + rule=rule, + tables=tables_doc, + fractional_bits=fractional_bits, + ) + canonical = document.canonical_bytes(built) + backend.check_document(canonical.decode("utf-8")) + tally, cell_tally = tallies + published = tally.pop(disclosure.FREE, 0) + return ProfileResult( + document=built, + canonical=canonical, + digest=document.digest_of(canonical), + tables=tuple(results), + published=published, + withheld=dict(sorted(tally.items())), + withheld_cells=dict(sorted(cell_tally.items())), + fractional_bits=fractional_bits, + ) + + +#: A date as a CSV, a JSON slice or a text column writes one: exactly ``YYYY-MM-DD``. +_DATE_TEXT = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}\Z") + + +def _dates_read(plan: ExposePlan, read: sources.SourceColumns) -> sources.SourceColumns: + """*read*, with each ``YYYY-MM-DD`` text of a column the contract types ``date`` as that day. + + A CSV holds every value as text, and the registry reads a timestamp only from + RFC 3339 with an offset (FLX-5022), so a ``date`` column read from a CSV was refused + where Parquet's, which arrives as a date, was read. The text is read strictly, as + the contract's type says: exactly ``YYYY-MM-DD`` and a real calendar day becomes that + day's midnight in UTC, as a date from a Parquet file or a database does + (:func:`sources.midnight_utc`). Any other value goes on as it came, for the reader to + read or refuse: DuckDB's own date reader is not used, because it reads ``infinity`` + and ``epoch`` as dates and drops a time of day. + """ + days = [column.name for column in plan.columns if column.date] + if not days: + return read + values = dict(read.values) + for name in days: + values[name] = [_as_day(value) for value in read.values[name]] + return replace(read, values=values) + + +def _as_day(value: Any) -> Any: + if isinstance(value, str) and _DATE_TEXT.match(value): + try: + return sources.midnight_utc(date.fromisoformat(value)) + except ValueError: # not a calendar day: the reader refuses it as written + return value + return value + + +def _default_block( + contract: Mapping[str, Any], plan: ExposePlan, backend: StatisticsBackend, fractional_bits: int +) -> Tuple[Optional[DeclaredBlock], Tuple[Tuple[str, str], ...]]: + """The block the profiler declares for *plan* from the contract alone, read by the registry. + + ``(None, ...)`` when the contract declares nothing to answer. The block is the + profiler's own, so a grammar refusal of it is a defect, and says so. + """ + expose = next( + item for item in iter_exposes(dict(contract)) if item.get("exposeId") == plan.expose_id + ) + planned = defaults.default_plan(expose, plan.columns, backend.enum_levels, fractional_bits) + if planned.block is None: + return None, planned.uncovered + try: + declared = backend.read_block(planned.block, contract, plan.expose_id) + except BlockRefused as exc: + raise ProfileError( + "profile_default_block_refused", + f"the statistics fluid profile declares for exposes[{plan.expose_id}] are refused " + f"by flux-stats' fidelity grammar: {exc}", + { + "read": "nothing: no record was read", + "next": "report this with your forge-cli and flux-stats versions", + }, + ) from None + _check_declared_columns(declared, plan) + block = replace(declared, every_declared_group=True, declared_ranges=planned.ranges) + return block, planned.uncovered + + +def _refuse_repeated(exposes: Sequence[ExposeSource]) -> None: + """Refuse an expose named twice: one document holds each table once.""" + seen = set() + for each in exposes: + if each.expose_id in seen: + raise ProfileError( + "profile_expose_repeated", + f"exposes[{each.expose_id}] is named more than once", + {"read": "nothing: no record was read", "next": "name each expose once"}, + ) + seen.add(each.expose_id) + + +def _plan_with_columns(contract: Mapping[str, Any], expose_id: str) -> ExposePlan: + """:func:`plan_expose`, refused when it leaves no column to profile.""" plan = plan_expose(contract, expose_id) if not plan.columns: raise ProfileError( @@ -473,15 +729,48 @@ def profile_expose( f"no column of exposes[{expose_id}] has a type fluid profile reads", {"not profiled": "; ".join(f"{name}: {reason}" for name, reason in plan.skipped)}, ) - declared, fractional_bits = _declared_block( - bundle, contract, expose_id, plan, backend, fractional_bits - ) - read = sources.read_columns( - source, [column.name for column in plan.columns], table=table, cwd=cwd - ) - tally: "Counter[str]" = Counter() - cell_tally: "Counter[str]" = Counter() - replaced = _ungrouped_means(declared) + return plan + + +def _one_grid( + grids: Mapping[str, int], blocks: Mapping[str, BundleBlock], requested: Optional[int] +) -> int: + """The one grid of the document: the blocks' (they must agree), else *requested* or 16. + + :func:`_declared_block` has already refused a *requested* grid a block disagrees with. + """ + if not grids: + return DEFAULT_FRACTIONAL_BITS if requested is None else requested + first, *others = grids.items() + for expose_id, grid in others: + if grid != first[1]: + raise ProfileError( + "profile_fractional_bits_conflict", + f"the fidelity blocks of exposes[{first[0]}] ({blocks[first[0]].simulation}) " + f"and exposes[{expose_id}] ({blocks[expose_id].simulation}) declare " + f"encoding.fractionalBits {first[1]} and {grid}", + { + "read": "nothing: no record was read", + "next": "one profile document has one grid: declare the same in both blocks", + }, + ) + return first[1] + + +def _profile_table( + plan: ExposePlan, + read: sources.SourceColumns, + declared: Optional[DeclaredBlock], + backend: StatisticsBackend, + rule: disclosure.DisclosureRule, + tallies: Tuple["Counter[str]", "Counter[str]"], + fractional_bits: int, +) -> Tuple[Dict[str, Any], TableResult]: + """One table of the document, from one read of its records, and its summary.""" + tally, cell_tally = tallies + binned = _binned_numbers(declared, plan) + replaced = _ungrouped_means(declared) | binned + counted = _counted_levels(declared) columns_doc = { column.name: _column( column, @@ -492,6 +781,8 @@ def profile_expose( tally, fractional_bits, column.name in replaced, + (frozenset({"distinctCount"}) if column.name in counted else frozenset()) + | (_BINNED_WITHHELD if column.name in binned else frozenset()), ) for column in plan.columns } @@ -506,29 +797,9 @@ def profile_expose( row_count = _row_count(read.row_count, rule, tally) if row_count is not None: table_doc["rowCount"] = row_count - if producer is None: - from fluid_build import __version__ - - producer = document.producer_uri(__version__) - built = document.build_document( - producer=producer, - rule=rule, - tables={expose_id: table_doc}, - fractional_bits=fractional_bits, - ) - canonical = document.canonical_bytes(built) - backend.check_document(canonical.decode("utf-8")) - published = tally.pop(disclosure.FREE, 0) all_published, none_published, some_published = _statistic_summaries(columns_doc) - return ProfileResult( - document=built, - canonical=canonical, - digest=document.digest_of(canonical), + return table_doc, TableResult( plan=plan, - published=published, - withheld=dict(sorted(tally.items())), - withheld_cells=dict(sorted(cell_tally.items())), - fractional_bits=fractional_bits, statistics=all_published, withheld_statistics=none_published, partly_withheld_statistics=some_published, @@ -579,6 +850,51 @@ def _declared_block( return declared, declared.fractional_bits +#: The metrics withheld beside a histogram the contract's own block declares on a number +#: column's declared range (``_binned_numbers``), whatever their values. +_BINNED_WITHHELD = frozenset({"sum", "min", "max", "quantiles", "distinctCount"}) + + +def _binned_numbers(declared: Optional[DeclaredBlock], plan: ExposePlan) -> FrozenSet[str]: + """The number columns the contract's own block gives a histogram over a declared range. + + The bins of a declared range are narrow, often one value each, so an exact ``sum`` is + the bins' counts weighed by their values, and a published ``min``, ``max``, + ``quantiles`` value or ``distinctCount`` places records in them: beside the cells the + check publishes, each gives a withheld one back. So those metrics are withheld under + ``D`` and no ``mean/1`` (the sum on the grid) is formed. ``tests/profiler/ + test_disclosure_oracle.py`` found it, and a quantile on a type's ladder, whose bins are + wide, keeps them all. + """ + if declared is None or not declared.every_declared_group: + return frozenset() + numbers = {column.name for column in plan.columns if column.domain == NUMBER} + return frozenset( + each.column + for each in declared.statistics + if each.column in numbers and each.kind == "marginalHistogram" + ) + + +def _counted_levels(declared: Optional[DeclaredBlock]) -> FrozenSet[str]: + """The columns whose levels a level histogram counts, or a grouped statistic groups by. + + Their ``distinctCount`` is withheld: the number of levels that hold a record says how + many of the histogram's withheld cells, or of its groups, hold none, and beside the + cells the check lets through it can pin a cell of one record or of none (found by + ``tests/profiler/test_disclosure_oracle.py``). + """ + if declared is None: + return frozenset() + found = set() + for statistic in declared.statistics: + if statistic.kind == "levelHistogram": + found.add(statistic.column) + if statistic.group_by is not None: + found.add(statistic.group_by) + return frozenset(found) + + def _ungrouped_means(declared: Optional[DeclaredBlock]) -> FrozenSet[str]: """The columns whose ungrouped mean the block declares: their fixed ``mean/1`` is the block's.""" if declared is None: @@ -636,6 +952,7 @@ def _declare_statistics( the column's other statistics (:func:`_edge_decisions`). *tallies* count the values and the histogram cells the checks withhold, by the check. """ + _check_declared_ranges(declared, read, backend) rows: Dict[Tuple[str, ...], List[Dict[str, Any]]] = {} left_out: List[Tuple[str, str]] = [] formed: List[Tuple[DeclaredStatistic, MeasuredStatistic]] = [] @@ -649,6 +966,12 @@ def _declare_statistics( try: measured = backend.measure(declared, statistic, rows[statistic.columns]) except MeasureRefused as exc: + if declared.every_declared_group: # the block the contract's declarations give + raise ProfileError( + "profile_record_outside_declared_domain", + f"{statistic.description}: {exc}", + {"written": "nothing", "next": _OUTSIDE_DECLARED_DOMAIN}, + ) from None raise ProfileError( "profile_bundle_record_refused", f"{statistic.description}: {exc}", @@ -663,13 +986,50 @@ def _declare_statistics( formed.append((statistic, measured)) decided = _edge_decisions(formed, backend, rule) for position, (statistic, measured) in enumerate(formed): + records = None + if statistic.group_by is not None: + records = sum(1 for value in read.values[statistic.column] if value is not None) entry = _declared_entry( - statistic, measured, rule, tallies, fractional_bits, decided.get(position) + statistic, + measured, + rule, + tallies, + fractional_bits, + decided.get(position), + records=records, ) columns_doc[statistic.column].setdefault("statistics", []).append(entry) return left_out +#: What to do about a record outside what the contract declares for its column. +_OUTSIDE_DECLARED_DOMAIN = ( + "the records and the contract disagree: fix the records, or the contract's column " + "type, enum levels or validationRules range, then profile" +) + + +def _check_declared_ranges( + declared: DeclaredBlock, read: sources.SourceColumns, backend: StatisticsBackend +) -> None: + """Refuse a value outside the range its column's validationRules declare, exactly. + + A histogram's bins over the range are rounded outward to the grid, which can be + coarser than the column's own scale, so the registry, which checks a value against + the bins, would count one just past a bound in an end bin. The value is never shown. + """ + for name, declared_range in declared.declared_ranges.items(): + for index, value in enumerate(read.values[name]): + exact = backend.read_value("number", value, column=name, row_index=index) + if exact is not None and not declared_range.holds(exact): + raise ProfileError( + "profile_record_outside_declared_domain", + f"column {name!r} row {index}: value outside the range its " + "validationRules declare", + {"written": "nothing", "next": _OUTSIDE_DECLARED_DOMAIN}, + ) + + #: The declared statistics that count records between edges of their column (FLUX ADR #: 0012), in the order a column's call to the output check lists them. _EDGE_KINDS = ("marginalHistogram", "levelHistogram", "quantile") @@ -802,21 +1162,27 @@ def _declared_entry( tallies: Tuple["Counter[str]", "Counter[str]"], fractional_bits: int, decision: Optional[EdgeDecision], + records: Optional[int] = None, ) -> Dict[str, Any]: """One declared statistic as the document writes it: every group checked, none guessed at. A histogram's or a quantile's groups come from the *decision* of the output check on its column, and a mean's and a support's from their own checks. *tallies* count the - groups and the histogram cells that are withheld, by the check. + groups and the histogram cells that are withheld, by the check. The ``sampleSize`` + band is of *records* where given, else of the records the groups hold. A grouped + statistic is given its column's non-null records, as its column's ungrouped figures + count them: its groups leave out each record whose ``groupBy`` key is null, so their + band, beside the column's own counts, would give the number of those back. """ groups = measured.groups if decision is not None: written = _edge_checked_groups(statistic, measured, rule, tallies, decision) else: written = _own_checked_groups(statistic, measured, rule, tallies[0], fractional_bits) + held = sum(group.k for group in groups) if records is None else records entry: Dict[str, Any] = { "definition": statistic.identifier, - "sampleSize": disclosure.sample_size(sum(group.k for group in groups), rule), + "sampleSize": disclosure.sample_size(held, rule), "disclosure": rule.name, **statistic.shape, "groups": written, @@ -994,7 +1360,10 @@ def _shape(statistic: Mapping[str, Any]) -> Optional[str]: if "level" in statistic: parts.append(f"at level {statistic['level']}") if "bins" in statistic: - parts.append(f"over bins {', '.join(statistic['bins'])}") + bins = statistic["bins"] + # A ladder has scores of edges: their number says which statistic it is. + shown = ", ".join(bins) if len(bins) <= 12 else f"{len(bins) - 1} bins" + parts.append(f"over bins {shown}" if len(bins) <= 12 else f"over {shown}") return " ".join(parts) or None diff --git a/fluid_build/profiler/sources.py b/fluid_build/profiler/sources.py index db20df4b..52b6f320 100644 --- a/fluid_build/profiler/sources.py +++ b/fluid_build/profiler/sources.py @@ -16,7 +16,8 @@ * a CSV or Parquet file, through DuckDB, inside the sandbox ``fluid init --discover`` reads with (:mod:`fluid_build.providers._duckdb_sandbox`), allowed that one file and nothing else. A CSV is read with every column as text - (``all_varchar``), so a decimal arrives as the digits the file holds. + (``all_varchar``), so a decimal arrives as the digits the file holds, and a date + as its text, which the runner reads as the contract's ``date`` type says. * a ``postgres://``, ``mysql://`` or ``sqlite:///`` database, attached by :func:`fluid_build.cli.discover._jdbc_introspect.attach_jdbc_source`, the one place ``fluid init --discover`` opens such a source. @@ -57,6 +58,7 @@ "SourceError", "describe_source", "local_file", + "midnight_utc", "read_columns", ] @@ -108,11 +110,19 @@ def _scheme(source: str) -> str: def describe_source(source: str) -> str: - """*source* as it may be shown: a database URI with its password masked.""" + """*source* as it may be shown: a database URI with its password masked. + + A URI after a leading ``=`` is masked too, so a ``--source`` paired with an + exposeId, or with a mistyped one, never shows its password. A URI's own ``=`` (its + query) sits after ``://``. + """ if _scheme(source) in DATABASE_SCHEMES: from fluid_build.cli.discover._jdbc_introspect import _masked_uri return _masked_uri(source) + head, sep, rest = source.partition("=") + if sep and "://" not in head and "://" in rest: + return f"{head}={describe_source(rest)}" return source @@ -219,11 +229,16 @@ def _rfc3339(moment: datetime) -> str: ) +def midnight_utc(day: date) -> str: + """A date as the registry reads it: that day's midnight in UTC, in RFC 3339.""" + return f"{day.year:04d}-{day.month:02d}-{day.day:02d}T00:00:00.000Z" + + def _wire(value: Any) -> Any: if isinstance(value, datetime): return _rfc3339(value) if isinstance(value, date): - return f"{value.year:04d}-{value.month:02d}-{value.day:02d}T00:00:00.000Z" + return midnight_utc(value) if isinstance(value, uuid.UUID): return str(value) return value diff --git a/fluid_build/providers/quality_engine.py b/fluid_build/providers/quality_engine.py index 8bd86be2..e873e259 100644 --- a/fluid_build/providers/quality_engine.py +++ b/fluid_build/providers/quality_engine.py @@ -20,6 +20,7 @@ import math import re from dataclasses import dataclass +from fractions import Fraction from typing import Any, Callable, Dict, List, Optional, Tuple from fluid_build.providers._sql_safety import quote_string_literal_for @@ -135,25 +136,48 @@ def extract_reference(rule: Dict[str, Any], columns: Any = None) -> Optional[Tup _RANGE_AND = re.compile(r"\s+and\s+", re.IGNORECASE) -def parse_range(constraint: Any) -> List[Tuple[str, float]]: - """The bounds of a ``range`` validationRule: ``">= 0"``, ``">= 0 and <= 100"`` - or ``"between 0 and 100"`` (``[]`` for anything else).""" +#: The longest number an exact range bound is read from: a longer text is not a bound +#: anyone wrote, and ``Fraction`` refuses an integer past 4,300 digits. +_MOST_RANGE_DIGITS = 1000 + + +def _range_parts(constraint: Any) -> List[Tuple[str, str]]: + """A ``range`` validationRule as ``(operator, number text)`` pairs; ``[]`` for anything else.""" if not isinstance(constraint, str): return [] m = _RANGE_BETWEEN.match(constraint) if m: - bounds = [(">=", float(m.group(1))), ("<=", float(m.group(2)))] - else: - bounds = [] - for part in _RANGE_AND.split(constraint.strip()): - m = _RANGE_BOUND.match(part) - if not m: - return [] - bounds.append((m.group(1), float(m.group(2)))) + return [(">=", m.group(1)), ("<=", m.group(2))] + parts = [] + for part in _RANGE_AND.split(constraint.strip()): + m = _RANGE_BOUND.match(part) + if not m: + return [] + parts.append((m.group(1), m.group(2))) + return parts + + +def parse_range(constraint: Any) -> List[Tuple[str, float]]: + """The bounds of a ``range`` validationRule: ``">= 0"``, ``">= 0 and <= 100"`` + or ``"between 0 and 100"`` (``[]`` for anything else).""" + bounds = [(operator, float(text)) for operator, text in _range_parts(constraint)] # A bound too large for a float would reach SQL as ``inf``: not a number. return bounds if all(math.isfinite(n) for _, n in bounds) else [] +def parse_range_exact(constraint: Any) -> List[Tuple[str, Fraction]]: + """:func:`parse_range`'s bounds as exact numbers, for a reader that must not round them. + + ``fluid profile`` cuts a declared range into bins (``profiler/defaults.py``), and a + float would move an edge. ``[]`` for anything :func:`parse_range` does not read, and + for a number longer than :data:`_MOST_RANGE_DIGITS`. + """ + parts = _range_parts(constraint) + if any(len(text) > _MOST_RANGE_DIGITS for _, text in parts): + return [] + return [(operator, Fraction(text)) for operator, text in parts] + + @dataclass(frozen=True) class ValueDomain: """What a valid value of one column is: the rule's own value list, or else diff --git a/pyproject.toml b/pyproject.toml index a5b73af3..b39a7f52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -947,6 +947,7 @@ module = [ # Any is a silent leak. "fluid_build.profiler._stats", "fluid_build.profiler.bundle", + "fluid_build.profiler.defaults", "fluid_build.profiler.disclosure", "fluid_build.profiler.document", "fluid_build.profiler.runner", diff --git a/tests/fluid_sql/test_render_cel.py b/tests/fluid_sql/test_render_cel.py index e863f1f4..a9aef08f 100644 --- a/tests/fluid_sql/test_render_cel.py +++ b/tests/fluid_sql/test_render_cel.py @@ -91,6 +91,9 @@ def test_a_name_that_is_not_a_cel_identifier_is_indexed() -> None: ("price + 1 > 1", "double column"), ("ROUND(qty, 0) = 1", "ROUND"), ("DAYS_BETWEEN(day, day) > 1", "DAYS_BETWEEN"), + ("ADD_DAYS(day, 1) = day", "ADD_DAYS"), + ("ADD_MONTHS(day, qty) = day", "ADD_MONTHS"), + ("ADD_YEARS(day, 1) = day", "ADD_YEARS"), ("day = DATE '2026-01-01'", "date"), ("total > 1", "decimal"), ("qty > 1.5", "not an integer"), diff --git a/tests/fluid_sql/test_render_dialects.py b/tests/fluid_sql/test_render_dialects.py index 37ba76bd..c8c26170 100644 --- a/tests/fluid_sql/test_render_dialects.py +++ b/tests/fluid_sql/test_render_dialects.py @@ -508,6 +508,104 @@ def test_month_and_year_difference_spelling_per_dialect() -> None: assert months.count("CASE WHEN") >= 3 +def _guard(unit: str, d: str, k: str, dialect: str) -> str: + """The count made NULL when the result would leave 0001-01-01..9999-12-31.""" + year = f"EXTRACT(YEAR FROM {d})" + if unit == "day": + days = { + "duckdb": lambda a, b: f"date_sub('day', {a}, {b})", + "snowflake": lambda a, b: f"DATEDIFF(day, {a}, {b})", + "bigquery": lambda a, b: f"DATE_DIFF({b}, {a}, DAY)", + "databricks": lambda a, b: f"datediff({b}, {a})", + "postgres": lambda a, b: f"({b} - {a})", + "athena": lambda a, b: f"date_diff('day', {a}, {b})", + }[dialect] + low, high = days(d, "DATE '0001-01-01'"), days(d, "DATE '9999-12-31'") + elif unit == "month": + index = f"({year} * 12 + EXTRACT(MONTH FROM {d}) - 1)" + low, high = f"(12 - {index})", f"(119999 - {index})" + else: + low, high = f"(1 - {year})", f"(9999 - {year})" + return f"(CASE WHEN {k} BETWEEN {low} AND {high} THEN {k} END)" + + +def test_date_addition_spelling_per_dialect() -> None: + cols = {"a": "date", "k": "int"} + for dialect in fs.DIALECTS: + q = fs.column_reader(dialect, "t") + d, k = q(None, "a"), q(None, "k") + for unit, name in n.DATE_ADD_FUNCTIONS.items(): + guard = _guard(unit, d, k, dialect) + whole = { + "bigquery": f"CAST({guard} AS INT64)", + "databricks": f"CAST({guard} AS INT)", + "athena": f"CAST({guard} AS BIGINT)", + } + whole_sql = whole.get(dialect, f"CAST({guard} AS INTEGER)") + upper = unit.upper() + want = { + "duckdb": { + "day": f"({d} + {whole_sql})", + "month": f"CAST({d} + to_months({whole_sql}) AS DATE)", + "year": f"CAST({d} + to_years({whole_sql}) AS DATE)", + }, + "postgres": { + "day": f"({d} + {whole_sql})", + "month": f"CAST({d} + {whole_sql} * INTERVAL '1 month' AS DATE)", + "year": f"CAST({d} + {whole_sql} * INTERVAL '1 year' AS DATE)", + }, + "snowflake": {u: f"DATEADD({u}, {guard}, {d})" for u in n.DATE_ADD_FUNCTIONS}, + "bigquery": { + u: f"DATE_ADD({d}, INTERVAL {whole_sql} {u.upper()})" + for u in n.DATE_ADD_FUNCTIONS + }, + "databricks": { + "day": f"date_add({d}, {whole_sql})", + "month": f"add_months({d}, {whole_sql})", + "year": f"add_months({d}, {whole_sql} * 12)", + }, + "athena": { + "day": f"date_add('day', {whole_sql}, {d})", + "month": f"({d} + {whole_sql} * INTERVAL '1' MONTH)", + "year": f"({d} + {whole_sql} * INTERVAL '1' YEAR)", + }, + }[dialect][unit] + sql = render(f"{name}(a, k)", cols, dialect) + assert sql == want, (dialect, name, upper) + parses(sql, dialect) + # Snowflake's own ADD_MONTHS keeps month ends, which FLUID SQL/1 does not. + assert "ADD_MONTHS" not in render("ADD_MONTHS(a, k)", cols, "snowflake") + + +def test_a_deeply_nested_date_addition_hits_the_cap_not_a_hang() -> None: + text = "a" + for _ in range(30): + text = f"ADD_DAYS({text}, 1)" # every level prints the one below it three times + assert len(text) <= fs.MAX_CODE_POINTS + start = time.perf_counter() + with pytest.raises(fs.RenderTooLarge): + render(text, {"a": "date"}, "postgres") + assert time.perf_counter() - start < 5.0 # linear work; slack for a loaded runner + + +@pytest.mark.parametrize("unit", ["DAYS", "MONTHS", "YEARS"]) +@pytest.mark.parametrize("sign", ["", "-"]) +def test_duckdb_gives_null_for_a_count_far_beyond_its_integer_type(unit: str, sign: str) -> None: + duckdb = pytest.importorskip("duckdb") + cols = {"a": "date", "k": "decimal(38,0)"} + sql = render(f"ADD_{unit}(a, k)", cols, "duckdb") + con = duckdb.connect() + try: + for day in ("2026-11-16", "0001-01-01", "9999-12-31"): + query = ( + f"SELECT {sql} FROM (SELECT CAST('{day}' AS DATE) AS a, " + f"CAST('{sign}1' || repeat('0', 30) AS DECIMAL(38,0)) AS k) AS t" + ) + assert con.execute(query.replace('"t".', "")).fetchone()[0] is None + finally: + con.close() + + def test_an_unknown_dialect_is_refused() -> None: tree, checked = compile_text("1 + 1", {}) with pytest.raises(ValueError): diff --git a/tests/fluid_sql/test_vectors.py b/tests/fluid_sql/test_vectors.py index 41832164..7f5e6753 100644 --- a/tests/fluid_sql/test_vectors.py +++ b/tests/fluid_sql/test_vectors.py @@ -147,3 +147,68 @@ def test_the_fill_script_has_nothing_to_fill(capsys: pytest.CaptureFixture[str]) def test_vectors_round_trip_through_json() -> None: assert json.loads(json.dumps(DOCUMENT)) == DOCUMENT + + +# -- ADD_DAYS, ADD_MONTHS, ADD_YEARS (EXPR-SPEC-3) ---------------------------------------- + +_BIG = 10**30 + + +@pytest.mark.parametrize("unit", ["day", "month", "year"]) +@pytest.mark.parametrize("count", [_BIG, -_BIG, 2**63, -(2**63) - 1, 10**9, -(10**9)]) +def test_a_huge_count_gives_null_never_an_error(unit: str, count: int) -> None: + import datetime + from fractions import Fraction + + for day in (datetime.date(2000, 2, 29), datetime.date.min, datetime.date.max): + tree = _nodes.DateAdd(unit, _nodes.Column("d"), _nodes.Column("k")) + assert fs.evaluate(tree, {"d": day, "k": Fraction(count)}) is None + + +@pytest.mark.parametrize( + "unit, day, count, want", + [ + ("day", "2000-01-01", 2921939, "9999-12-31"), + ("day", "2000-01-01", 2921940, None), + ("day", "2000-01-01", -730119, "0001-01-01"), + ("day", "2000-01-01", -730120, None), + ("month", "9999-12-31", 0, "9999-12-31"), + ("month", "9999-12-31", 1, None), + ("month", "0001-01-31", -1, None), + ("month", "0001-01-31", 1, "0001-02-28"), + ("month", "2001-02-28", 1, "2001-03-28"), + ("year", "2000-02-29", 100, "2100-02-28"), + ("year", "2000-02-29", 400, "2400-02-29"), + ("year", "0001-02-28", 9998, "9999-02-28"), + ("year", "0001-02-28", 9999, None), + ], +) +def test_date_addition_edges(unit: str, day: str, count: int, want: Any) -> None: + import datetime + from fractions import Fraction + + tree = _nodes.DateAdd(unit, _nodes.Column("d"), _nodes.Column("k")) + got = fs.evaluate(tree, {"d": datetime.date.fromisoformat(day), "k": Fraction(count)}) + assert got == (None if want is None else datetime.date.fromisoformat(want)) + + +def test_the_date_addition_names_and_hints() -> None: + assert _nodes.DATE_ADD_FUNCTIONS == { + "day": "ADD_DAYS", + "month": "ADD_MONTHS", + "year": "ADD_YEARS", + } + assert fs.FUNCTIONS[-3:] == ("ADD_DAYS", "ADD_MONTHS", "ADD_YEARS") + for text in ("INTERVAL(1)", "DATEADD(day, 1, d)", "DATE_ADD(d, 1)", "TIMESTAMPADD(d, 1)"): + with pytest.raises(fs.FluidSqlError) as caught: + fs.parse(text) + assert "ADD_DAYS(d, n), ADD_MONTHS(d, n) or ADD_YEARS(d, n)" in caught.value.detail + with pytest.raises(fs.FluidSqlError) as caught: + fs.parse("DATE_SUB(a, b)") + assert "DAYS_BETWEEN" in caught.value.detail + + +def test_a_date_addition_prints_its_canonical_text() -> None: + tree = fs.parse("add_months( issue_date ,1+1 )") + assert fs.canonical(tree) == "ADD_MONTHS(issue_date, 1 + 1)" + assert [type(c).__name__ for c in tree.children()] == ["Column", "Arith"] diff --git a/tests/profiler/_fake_backend.py b/tests/profiler/_fake_backend.py index 9f73e8ff..2b53997f 100644 --- a/tests/profiler/_fake_backend.py +++ b/tests/profiler/_fake_backend.py @@ -15,6 +15,11 @@ ``tests/profiler/test_flux_stats_backend.py`` runs the same profiles through the real registry wherever flux-stats is installed, and ``[dev]`` installs it in every CI lane, so this double cannot drift from it unnoticed. + +The fidelity grammar has no stand-in: it is flux-stats' own (ADR 0009), and the +profile every run answers without ``--bundle`` is a fidelity block the profiler +declares from the contract (``fluid_build/profiler/defaults.py``). So ``read_block``, +``measure``, ``check_edge_counts`` and ``enum_levels`` hand the work to flux-stats. """ from __future__ import annotations @@ -178,6 +183,29 @@ class FakeBackend: def __init__(self) -> None: self.checked: List[str] = [] + self._registry: Any = None + + def _real(self) -> Any: + if self._registry is None: + # Built directly: a test that puts this double in place of load_backend would + # otherwise get this double back. + from fluid_build.profiler._stats import _FluxStatsBackend + + self._registry = _FluxStatsBackend() + return self._registry + + def enum_levels(self, declared_type: str) -> Tuple[str, ...]: + levels: Tuple[str, ...] = self._real().enum_levels(declared_type) + return levels + + def read_block(self, raw_block: Any, contract_document: Any, expose_id: str) -> Any: + return self._real().read_block(raw_block, contract_document, expose_id) + + def measure(self, block: Any, statistic: Any, rows: Any) -> Any: + return self._real().measure(block, statistic, rows) + + def check_edge_counts(self, statistics: Any, **checks: Any) -> Any: + return self._real().check_edge_counts(statistics, **checks) def metrics(self) -> Tuple[FakeMetric, ...]: return METRICS @@ -206,12 +234,17 @@ def check_document(self, text: str) -> None: self.checked.append(text) document = json.loads(text, parse_float=_refuse_float) schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) - errors = sorted( - jsonschema.Draft202012Validator(schema).iter_errors(document), - key=lambda error: list(error.absolute_path), - ) - if errors: - raise DocumentRefused("; ".join(error.message for error in errors[:3])) + if document.get("_schemaURL") == schema["$id"]: # format 1-0-0: the vendored schema + errors = sorted( + jsonschema.Draft202012Validator(schema).iter_errors(document), + key=lambda error: list(error.absolute_path), + ) + if errors: + raise DocumentRefused("; ".join(error.message for error in errors[:3])) + # Every document is also read back by the registry's own reader: a schema does not + # hold its rules (levels sorted, a null cell only where withheld, groups in order), + # and 1-1-0 (a quantile/2) has no vendored schema here. + self._real().check_document(text) def read_budget(self) -> object: return object() diff --git a/tests/profiler/test_bundle_flux_stats.py b/tests/profiler/test_bundle_flux_stats.py index ac44fb4b..2c2c2ffd 100644 --- a/tests/profiler/test_bundle_flux_stats.py +++ b/tests/profiler/test_bundle_flux_stats.py @@ -289,7 +289,7 @@ def test_a_rare_support_member_withholds_the_group_and_siblings_of_a_grouped_mea # The ungrouped mean needs no secondary suppression. assert _entry(result, "amount", "mean/1")["groups"][0]["status"] == "F" # Three siblings of the mean, and the quantile beside the histogram of its column. - assert result.withheld["D"] == 4 + assert result.withheld["D"] == 4 + 1 # + the groupBy column's distinctCount assert ("amount", "mean/1", "market", None) not in result.statistics assert ("amount", "mean/1", None, None) in result.statistics assert ("market", "support/1", None, None) in result.withheld_statistics @@ -378,7 +378,7 @@ def test_a_group_withheld_for_its_size_withholds_its_siblings_in_a_grouped_histo ("GB", "C"), ("IE", "C"), ] - assert result.withheld["D"] == 3 + assert result.withheld["D"] == 3 + 1 # + the groupBy column's distinctCount # The column's own histogram is no group of the groupBy. assert _entry(result, "amount", "marginalHistogram/1")["groups"][0]["status"] == "F" @@ -399,7 +399,7 @@ def test_a_group_withheld_for_its_size_withholds_its_siblings_in_a_grouped_quant grouped = _entry(result, "amount", "quantile/1", "market") # Every other market's median passes its own checks, and goes with the small one. assert {g["status"] for g in grouped["groups"]} == {"C"} - assert result.withheld["D"] == 3 + assert result.withheld["D"] == 3 + 1 # + the groupBy column's distinctCount @pytest.mark.parametrize( @@ -637,7 +637,9 @@ def test_rows_no_group_holds_that_the_block_refuses_withhold_a_grouped_histogram result = _profile(tmp_path, rows, [by_market], market_null="exclude") grouped = _entry(result, "amount", "marginalHistogram/1", "market") assert {g["status"] for g in grouped["groups"]} == {"C"} - assert result.withheld["D"] == len(grouped["groups"]) + assert ( + result.withheld["D"] == len(grouped["groups"]) + 1 + ) # + the groupBy column's distinctCount def test_few_rows_no_group_holds_withhold_a_grouped_histogram(tmp_path): @@ -646,7 +648,9 @@ def test_few_rows_no_group_holds_withhold_a_grouped_histogram(tmp_path): result = _profile(tmp_path, _null_market_rows(3), [by_market], market_null="exclude") grouped = _entry(result, "amount", "marginalHistogram/1", "market") assert {g["status"] for g in grouped["groups"]} == {"C"} - assert result.withheld["D"] == len(grouped["groups"]) + assert ( + result.withheld["D"] == len(grouped["groups"]) + 1 + ) # + the groupBy column's distinctCount def test_a_quantile_whose_witness_too_few_records_hold_is_withheld(tmp_path): @@ -678,7 +682,7 @@ def test_a_record_no_group_holds_withholds_the_groups_of_a_grouped_mean(tmp_path ("GB", "D"), ("IE", "D"), ] - assert result.withheld["D"] == 3 + assert result.withheld["D"] == 3 + 1 # + the groupBy column's distinctCount assert _entry(result, "amount", "mean/1")["groups"][0]["status"] == "F" diff --git a/tests/profiler/test_cli_profile.py b/tests/profiler/test_cli_profile.py index 9a88d2c4..14970d16 100644 --- a/tests/profiler/test_cli_profile.py +++ b/tests/profiler/test_cli_profile.py @@ -115,6 +115,239 @@ def test_an_unknown_expose_is_named_with_the_ones_there_are(workspace, registry, assert "profile_expose_not_found" in out and "orders" in out +TWO_EXPOSES = """\ +fluidVersion: "0.7.7" +kind: DataProduct +id: crm.accounts +name: Accounts +exposes: + - exposeId: org + kind: table + binding: {platform: local, format: csv, location: {path: out/org.csv}} + contract: + schema: + - {name: org_id, type: int} + - exposeId: account + kind: table + binding: {platform: local, format: csv, location: {path: out/account.csv}} + contract: + schema: + - {name: account_id, type: int} + - {name: org_id, type: int} +""" + + +def test_several_exposes_without_paired_sources_are_refused_before_any_read( + workspace, registry, capsys +): + # argparse used to keep the last --expose, so org.csv was profiled as account. Several + # --expose flags now need one --source = each. + (workspace / "two.fluid.yaml").write_text(TWO_EXPOSES, encoding="utf-8") + (workspace / "org.csv").write_text("org_id\n1\n", encoding="utf-8") + rc = main( + [ + "profile", + "two.fluid.yaml", + "--expose", + "org", + "--expose", + "account", + "--source", + "org.csv", + ] + ) + out = " ".join(capsys.readouterr().out.split()) # the console may wrap + assert rc == 1 + assert "profile_source_unpaired" in out + assert "--expose is given 2 times (org, account)" in out + assert "--source =" in out + assert "no record was read" in out + assert registry.checked == [] + assert not list(workspace.glob("*.profile.json")) + + +def _two_sources(workspace): + (workspace / "two.fluid.yaml").write_text(TWO_EXPOSES, encoding="utf-8") + (workspace / "org.ndjson").write_text( + "".join(json.dumps({"org_id": str(n)}) + "\n" for n in range(1, 13)), encoding="utf-8" + ) + (workspace / "account.ndjson").write_text( + "".join( + json.dumps({"account_id": str(n), "org_id": str(1 + n % 12)}) + "\n" for n in range(40) + ), + encoding="utf-8", + ) + + +def test_the_whole_product_is_profiled_into_one_document(workspace, registry, capsys): + _two_sources(workspace) + rc = main( + [ + "profile", + "two.fluid.yaml", + "--source", + "account=account.ndjson", + "--source", + "org=org.ndjson", + ] + ) + out = " ".join(capsys.readouterr().out.split()) + assert rc == 0, out + written = workspace / "crm.accounts.profile.json" # the contract's id names it + document = json.loads(written.read_bytes()) + assert list(document["tables"]) == ["account", "org"] + assert document["tables"]["org"]["rowCount"] == 10 # 12, rounded to 5 + assert document["tables"]["account"]["rowCount"] == 40 + assert "Profiled 2 exposes (org, account): 3 columns" in out # the contract's order + assert registry.checked == [written.read_text(encoding="utf-8")] + + +def test_repeated_exposes_take_their_order_and_their_own_sources(workspace, registry, capsys): + _two_sources(workspace) + rc = main( + [ + "profile", + "two.fluid.yaml", + "--expose", + "account", + "--expose", + "org", + "--source", + "org=org.ndjson", + "--source", + "account=account.ndjson", + "--out", + "both.json", + ] + ) + out = " ".join(capsys.readouterr().out.split()) + assert rc == 0, out + assert "Profiled 2 exposes (account, org)" in out + document = json.loads((workspace / "both.json").read_bytes()) + assert set(document["tables"]["org"]["columns"]) == {"org_id"} + assert set(document["tables"]["account"]["columns"]) == {"account_id", "org_id"} + + +@pytest.mark.parametrize( + "arguments, event, words", + [ + ( + ["--expose", "org", "--expose", "account", "--source", "org=org.ndjson"], + "profile_source_missing", + "exposes[account] has no --source account=", + ), + ( + ["--expose", "org", "--source", "org=org.ndjson", "--source", "account=account.ndjson"], + "profile_source_unused", + "--source names exposes[account], which no --expose names", + ), + ( + ["--source", "org=org.ndjson", "--source", "account.ndjson"], + "profile_source_unpaired", + "--source account.ndjson names no expose", + ), + ( + ["--source", "org=org.ndjson", "--source", "org=account.ndjson"], + "profile_source_repeated", + "exposes[org] is given more than one --source", + ), + ( + ["--expose", "org", "--expose", "org", "--source", "org=org.ndjson"], + "profile_expose_repeated", + "--expose names 'org' twice", + ), + ( + ["--source", "org.ndjson", "--source", "account.ndjson"], + "profile_source_unpaired", + "--source is given 2 times", + ), + ( + [ + "--source", + "org=org.ndjson", + "--source", + "account=account.ndjson", + "--table", + "accounts", + ], + "profile_table_unpaired", + "--table names no expose", + ), + ], +) +def test_sources_that_do_not_pair_with_exposes_are_refused_before_any_read( + workspace, registry, capsys, arguments, event, words +): + _two_sources(workspace) + rc = main(["profile", "two.fluid.yaml", *arguments]) + out = " ".join(capsys.readouterr().out.split()) + assert rc == 1 + assert event in out and words in out, out + assert registry.checked == [] + assert not list(workspace.glob("*.profile.json")) + + +def test_a_paired_database_source_never_shows_its_password(workspace, registry, capsys): + pytest.importorskip("duckdb") + _two_sources(workspace) + secret = "pw-41871" # pragma: allowlist secret + rc = main( + [ + "profile", + "two.fluid.yaml", + "--source", + "org=org.ndjson", + "--source", + f"account=postgresql://reader:{secret}@127.0.0.1:9/crm", + "--table", + "account=public.account", + ] + ) + out = capsys.readouterr().out + assert rc == 1 + assert "profile_source_unreadable" in out + assert secret not in out + + +def test_the_document_cannot_replace_any_of_its_sources(workspace, registry, capsys): + _two_sources(workspace) + rc = main( + [ + "profile", + "two.fluid.yaml", + "--source", + "org=org.ndjson", + "--source", + "account=account.ndjson", + "--out", + "account.ndjson", + ] + ) + assert rc == 1 + assert "profile_out_is_the_source" in capsys.readouterr().out + + +def test_a_mistyped_exposeid_before_a_database_uri_never_shows_its_password( + workspace, registry, capsys +): + _two_sources(workspace) + secret = "pw-55210" # pragma: allowlist secret + rc = main( + [ + "profile", + "two.fluid.yaml", + "--source", + "org=org.ndjson", + "--source", + f"acount=postgresql://reader:{secret}@127.0.0.1:9/crm", + ] + ) + out = " ".join(capsys.readouterr().out.split()) + assert rc == 1 + assert "profile_source_unpaired" in out and "acount=postgresql://reader" in out + assert secret not in out + + def test_an_unreadable_source_is_reported_with_its_password_masked(workspace, registry, capsys): # The read itself is the [profile] extra's (DuckDB): without it the command stops # earlier, at "needs duckdb", as a lane without the extra (iac Stage 1) shows. @@ -263,13 +496,15 @@ def test_the_summary_names_each_statistic_once_with_its_columns_and_shape(): def test_the_summary_names_the_fidelity_statistics_and_their_grid(workspace, registry, capsys): assert main(["profile", "contract.fluid.yaml", "--source", "orders.ndjson"]) == 0 out = " ".join(capsys.readouterr().out.split()) - assert ( - "Fidelity statistics: mean/1 of 'amount', on fractionalBits 16 " - "(the fidelity block must declare the same)." in out - ) + assert "Fidelity statistics: mean/1 of 'amount'" in out + assert "on fractionalBits 16 (the fidelity block must declare the same)." in out written = json.loads((workspace / "orders.profile.json").read_bytes()) assert written["encoding"] == {"fractionalBits": 16, "rounding": "floor"} - (statistic,) = written["tables"]["orders"]["columns"]["amount"]["statistics"] + (statistic,) = [ + each + for each in written["tables"]["orders"]["columns"]["amount"]["statistics"] + if each["definition"] == "mean/1" + ] assert statistic["groups"] == [{"group": "", "k": 40, "Sx": str(620 * 2**16), "status": "F"}] @@ -286,7 +521,11 @@ def test_fractional_bits_names_the_grid(workspace, registry, capsys): assert "on fractionalBits 20 " in " ".join(capsys.readouterr().out.split()) written = json.loads((workspace / "orders.profile.json").read_bytes()) assert written["encoding"]["fractionalBits"] == 20 - (statistic,) = written["tables"]["orders"]["columns"]["amount"]["statistics"] + (statistic,) = [ + each + for each in written["tables"]["orders"]["columns"]["amount"]["statistics"] + if each["definition"] == "mean/1" + ] assert statistic["groups"][0]["Sx"] == str(620 * 2**20) diff --git a/tests/profiler/test_defaults.py b/tests/profiler/test_defaults.py new file mode 100644 index 00000000..330f349f --- /dev/null +++ b/tests/profiler/test_defaults.py @@ -0,0 +1,426 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""The statistics a profile carries without ``--bundle``, declared from the contract alone.""" + +from __future__ import annotations + +import json +from fractions import Fraction +from pathlib import Path + +import pytest + +from fluid_build.profiler import defaults +from fluid_build.profiler.runner import ProfileError, profile_expose +from tests.profiler._fake_backend import FakeBackend + +pytestmark = pytest.mark.unit + +pytest.importorskip("flux_stats.fidelity") + + +def _contract(columns, dimensions=()): + expose = { + "exposeId": "t", + "contract": {"schema": [{"name": n, "type": t, **extra} for n, t, extra in columns]}, + } + if dimensions: + expose["semantics"] = { + "dimensions": [{"name": name, "type": "categorical"} for name in dimensions] + } + return {"fluidVersion": "0.7.7", "kind": "DataProduct", "id": "p", "exposes": [expose]} + + +def _profile(tmp_path: Path, contract, rows, **options): + path = tmp_path / "t.ndjson" + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return profile_expose(contract, "t", source=str(path), backend=FakeBackend(), **options) + + +def _statistics(result, column): + return result.document["tables"]["t"]["columns"][column].get("statistics", []) + + +def _one(result, column, definition, **shape): + found = [ + each + for each in _statistics(result, column) + if each["definition"] == definition + and all(each.get(key) == value for key, value in shape.items()) + ] + assert len(found) == 1, (column, definition, shape, _statistics(result, column)) + return found[0] + + +STATUS = ("status", "enum('active', 'closed', 'suspended', 'unused')", {}) + + +def test_every_declared_level_is_a_cell_and_one_no_record_holds_is_withheld(tmp_path): + rows = [{"status": level} for level in ["active"] * 70 + ["closed"] * 25 + ["suspended"] * 5] + result = _profile(tmp_path, _contract([STATUS]), rows) + histogram = _one(result, "status", "levelHistogram/1") + assert histogram["levels"] == ["active", "closed", "suspended", "unused"] + (group,) = histogram["groups"] + # 'unused' holds no record and 'suspended' 5: both are small, and so together (5); the + # published total would form them from 'closed', so 'closed' is withheld beside them. + assert group["levelCounts"] == [70, None, None, None] + assert group["cellStatus"] == ["F", "C", "C", "C"] + + +def test_the_levels_listed_never_depend_on_the_records(tmp_path): + some = _profile(tmp_path, _contract([STATUS]), [{"status": "active"}] * 40) + other = _profile(tmp_path, _contract([STATUS]), [{"status": "closed"}] * 3) + assert ( + _one(some, "status", "levelHistogram/1")["levels"] + == _one(other, "status", "levelHistogram/1")["levels"] + ) + + +def test_a_level_the_type_does_not_list_refuses_the_profile_without_showing_it(tmp_path): + rows = [{"status": "active"}] * 20 + [{"status": "secret-level"}] + with pytest.raises(ProfileError) as refused: + _profile(tmp_path, _contract([STATUS]), rows) + assert refused.value.event == "profile_record_outside_declared_domain" + assert "secret-level" not in str(refused.value) + assert "row 20" in str(refused.value) + + +def test_a_declared_range_gets_a_histogram_on_bins_of_one_width(tmp_path): + score = ( + "score", + "int", + {"validationRules": [{"type": "range", "constraint": ">= 0 and <= 100"}]}, + ) + rows = [{"score": str(n % 101)} for n in range(400)] + result = _profile(tmp_path, _contract([score]), rows) + bins = [*(str(edge) for edge in range(0, 101, 10)), "100.5"] # 100 alone in the last bin + assert _one(result, "score", "marginalHistogram/1")["bins"] == bins + # The histogram alone: checked beside quantiles on the same bins, the two let a reader + # who knows the check tell worlds apart (tests/profiler/test_disclosure_oracle.py). + definitions = [s["definition"] for s in _statistics(result, "score")] + assert definitions == ["marginalHistogram/1"] + (group,) = _one(result, "score", "marginalHistogram/1")["groups"] + # 0 to 96 are held 4 times each and 97 to 100 three times: every bin of ten publishes, + # and 100, alone in the last bin with 3 records, is withheld with the bin beside it. + assert group["status"] == "F" + assert group["binCounts"][:9] == [40] * 9 + assert group["binCounts"][9:] == [None, None] + assert group["cellStatus"][9:] == ["C", "C"] + # Its sum, distinct count and quantiles are withheld, and it has no mean/1: each would + # give a withheld bin back from the published ones. + metrics = result.document["tables"]["t"]["columns"]["score"]["metrics"] + for name in ("sum", "distinctCount", "quantiles"): + assert metrics[name]["status"] == "D" and "value" not in metrics[name] + assert "values" not in metrics[name] + assert metrics["count"]["status"] == "F" + + +@pytest.mark.parametrize("bits", [16, 0]) # at 0 the last bin, [5, 6], holds 6 too +def test_a_value_outside_the_declared_range_refuses_the_profile(tmp_path, bits): + score = ( + "score", + "int", + {"validationRules": [{"type": "range", "constraint": "between 1 and 5"}]}, + ) + rows = [{"score": "3"}] * 20 + [{"score": "6"}] + with pytest.raises(ProfileError) as refused: + _profile(tmp_path, _contract([score]), rows, fractional_bits=bits) + assert refused.value.event == "profile_record_outside_declared_domain" + assert "'6'" not in str(refused.value) + assert "column type" in refused.value.context["next"] + + +@pytest.mark.parametrize( + "declared, bits, outside", + [ + ("decimal(10,6)", 16, "0.099995"), # the grid's step, 2^-16, is coarser than 10^-6 + ("decimal(10,6)", 16, "0.900005"), + ("decimal(5,2)", 0, "0.05"), + ], +) +def test_a_value_just_past_a_bound_is_refused_where_the_grid_is_coarser( + tmp_path, declared, bits, outside +): + # The bins are rounded outward to the grid, so an end bin reaches past the bound: each + # value is checked against the declared range itself. + rule = {"validationRules": [{"type": "range", "constraint": "between 0.1 and 0.9"}]} + contract = _contract([("share", declared, rule)]) + inside = [{"share": value} for value in ("0.1", "0.5", "0.9")] * 10 + _profile(tmp_path, contract, inside, fractional_bits=bits) # the bounds themselves are in + with pytest.raises(ProfileError) as refused: + _profile(tmp_path, contract, inside + [{"share": outside}], fractional_bits=bits) + assert refused.value.event == "profile_record_outside_declared_domain" + assert outside not in str(refused.value) and "row 30" in str(refused.value) + + +def test_a_strict_bound_is_not_reached(tmp_path): + rule = {"validationRules": [{"type": "range", "constraint": "> 0 and <= 10"}]} + contract = _contract([("score", "int", rule)]) + rows = [{"score": str(1 + n % 10)} for n in range(40)] + _profile(tmp_path, contract, rows) + with pytest.raises(ProfileError) as refused: + _profile(tmp_path, contract, rows + [{"score": "0"}]) + assert refused.value.event == "profile_record_outside_declared_domain" + + +@pytest.mark.parametrize( + "declared, constraint", + [ + ("int", ">= 0"), + ("decimal(10,2)", ">= 0"), + ("int", "<= 100"), + ("int", "between 0 and 1" + "0" * 30), # a top past the type's own bound + ], +) +def test_a_range_not_closed_inside_its_type_takes_the_types_ladder(tmp_path, declared, constraint): + # Ten bins over the rest of the type would hold every record in one of them and publish + # none: no histogram, so the column keeps its sum, its mean/1 and quantiles. + rule = {"validationRules": [{"type": "range", "constraint": constraint}]} + result = _profile( + tmp_path, + _contract([("amount", declared, rule)]), + [{"amount": str(n % 90)} for n in range(200)], + ) + assert {s["definition"] for s in _statistics(result, "amount")} == {"mean/1", "quantile/2"} + metrics = result.document["tables"]["t"]["columns"]["amount"]["metrics"] + assert metrics["sum"]["status"] == "F" + assert result.tables[0].uncovered == () + + +@pytest.mark.parametrize("rules", [7, True, "between 0 and 5", {"type": "range"}]) +def test_validation_rules_that_are_not_a_list_declare_no_range(tmp_path, rules): + contract = _contract([("n", "int", {"validationRules": rules})]) + result = _profile(tmp_path, contract, [{"n": str(n)} for n in range(30)]) + assert {s["definition"] for s in _statistics(result, "n")} == {"mean/1", "quantile/2"} + + +def test_without_a_declared_range_a_number_gets_quantiles_on_its_types_ladder(tmp_path): + result = _profile( + tmp_path, _contract([("age", "int", {})]), [{"age": str(18 + n % 60)} for n in range(500)] + ) + assert not [s for s in _statistics(result, "age") if s["definition"] == "marginalHistogram/1"] + quantile = _one(result, "age", "quantile/2", level="0.5") + assert quantile["bins"][0] == str(-(2**63)) and quantile["bins"][-1] == str(2**63 - 1) + assert {"0", "1", "2", "5", "10", "20", "50", "100"} <= set(quantile["bins"]) + (group,) = quantile["groups"] + # The median of 18..77 lies in [20, 50), with records below, in and above that bin. + assert group["status"] == "F" + assert quantile["bins"][group["quantileBin"]] == "20" + + +def test_the_bins_never_depend_on_the_records(tmp_path): + contract = _contract([("age", "int", {}), ("joined", "date", {})]) + young = _profile(tmp_path, contract, [{"age": "3", "joined": "1999-05-01"}] * 40) + old = _profile(tmp_path, contract, [{"age": "90", "joined": "2026-11-19"}] * 3) + for column in ("age", "joined"): + assert [s.get("bins") for s in _statistics(young, column)] == [ + s.get("bins") for s in _statistics(old, column) + ] + + +def test_a_date_gets_quantiles_on_a_ladder_of_years(tmp_path): + rows = [{"joined": f"20{10 + n % 15}-0{1 + n % 9}-1{n % 10}"} for n in range(300)] + result = _profile(tmp_path, _contract([("joined", "date", {})]), rows) + quantile = _one(result, "joined", "quantile/2", level="0.5") + assert len(quantile["bins"]) == 2 + 201 # the range's two ends and 1900 to 2100 + (group,) = quantile["groups"] + assert group["status"] == "F" + + +def test_an_unbounded_number_is_named_and_given_no_statistic(tmp_path): + result = _profile(tmp_path, _contract([("price", "decimal", {})]), [{"price": "1.5"}] * 20) + assert [s["definition"] for s in _statistics(result, "price")] == ["mean/1"] + (table,) = result.tables + assert table.uncovered == ( + ("price", "neither its type nor a validationRules range bounds it within 10^38"), + ) + + +@pytest.mark.parametrize( + "declared, rules, reason", + [ + ( + "decimal(60,0)", + [], + "neither its type nor a validationRules range bounds it within 10^38", + ), + ( + "decimal", + [{"type": "range", "constraint": "between 0 and " + "9" * 40}], + "wider than 10^38", + ), + ], +) +def test_a_number_too_wide_for_an_edge_gets_no_statistic_and_the_profile_stands( + tmp_path, declared, rules, reason +): + # Its edges would pass the 1,000 digits flux-stats writes out, refusing the profile. + contract = _contract([("v", declared, {"validationRules": rules})]) + result = _profile(tmp_path, contract, [{"v": "5"}] * 12) + assert [s["definition"] for s in _statistics(result, "v")] == ["mean/1"] + ((name, why),) = result.tables[0].uncovered + assert name == "v" and reason in why + + +def test_a_declared_grouping_lists_every_declared_level_as_a_group(tmp_path): + region = ("region", "enum('north', 'south', 'west')", {}) + rows = [ + {"status": status, "region": region} + for status, region in [("active", "north")] * 30 + [("closed", "south")] * 30 + ] + result = _profile(tmp_path, _contract([STATUS, region], dimensions=["region"]), rows) + grouped = _one(result, "status", "levelHistogram/1", groupBy="region") + # 'west' holds no record and is listed all the same; a group of none withholds its siblings. + assert [group["group"] for group in grouped["groups"]] == ["north", "south", "west"] + assert {group["status"] for group in grouped["groups"]} == {"C"} + # The grouping's own column is not grouped by itself. + assert not [s for s in _statistics(result, "region") if s.get("groupBy") == "region"] + + +def test_a_full_grouping_publishes_its_groups(tmp_path): + # Beside the column's own histogram a grouping publishes only when every cell of every + # group does (ADR 0012), so every level of both columns is full here. + region = ("region", "enum('north', 'south')", {}) + rows = [ + {"status": status, "region": region} + for status, region in [("active", "north")] * 40 + + [("closed", "north")] * 20 + + [("active", "south")] * 30 + + [("closed", "south")] * 30 + ] + status = ("status", "enum('active', 'closed')", {}) + result = _profile(tmp_path, _contract([status, region], dimensions=["region"]), rows) + grouped = _one(result, "status", "levelHistogram/1", groupBy="region") + assert [group["status"] for group in grouped["groups"]] == ["F", "F"] + assert [group["levelCounts"] for group in grouped["groups"]] == [[40, 20], [30, 30]] + + +def test_a_grouped_statistic_bands_its_columns_non_null_records(tmp_path): + # A record whose key is null is in no group. The band counts it all the same, as the + # column's own figures do: the records its groups hold, beside those, would say how + # many records hold a status and no region. + region = ("region", "enum('north', 'south')", {}) + status = ("status", "enum('active', 'closed')", {}) + rows = [{"status": "active", "region": "north"}] * 40 + [ + {"status": "closed", "region": None} + ] * 3 + result = _profile(tmp_path, _contract([status, region], dimensions=["region"]), rows) + grouped = _one(result, "status", "levelHistogram/1", groupBy="region") + ungrouped = _one(result, "status", "levelHistogram/1", groupBy=None) + assert grouped["sampleSize"] == ungrouped["sampleSize"] + held = sum(group["k"] for group in grouped["groups"] if group.get("k") is not None) + assert held <= 40 + + +def test_groupings_past_the_cell_budget_are_left_out_and_named(tmp_path): + # 101 levels by 100 keys is 10,100 cells: checking them together costs about the cube + # of the levels, so the grouping is left out, as the declared levels alone decide. + values = ", ".join(f"'v{n:03d}'" for n in range(101)) + keys = ", ".join(f"'k{n:03d}'" for n in range(100)) + columns = [("value", f"enum({values})", {}), ("key", f"enum({keys})", {})] + result = _profile( + tmp_path, _contract(columns, dimensions=["key"]), [{"value": "v000", "key": "k000"}] * 20 + ) + assert not [s for s in _statistics(result, "value") if s.get("groupBy") == "key"] + assert _one(result, "value", "levelHistogram/1", groupBy=None) + ((name, why),) = result.tables[0].uncovered + assert name == "value" and "10,000 cells" in why + + +def test_with_a_bundle_no_default_statistic_is_declared(tmp_path): + block = { + "encoding": {"fractionalBits": 16}, + "records": {"min": 1, "max": 10}, + "budget": {"maxChecksPerState": 1}, + "columns": {"status": {"support": "declared", "maxCardinality": 4}}, + "properties": [ + {"property": "support", "column": "status", "tolerance": {"type": "count", "max": 0}} + ], + } + from fluid_build.profiler.bundle import BundleBlock + + result = _profile( + tmp_path, + _contract([STATUS, ("age", "int", {})]), + [{"status": "active", "age": "30"}] * 20, + bundle=BundleBlock(raw=block, simulation="s.flux.yml", expose_id="t"), + ) + assert [s["definition"] for s in _statistics(result, "status")] == ["support/1"] + assert [s["definition"] for s in _statistics(result, "age")] == ["mean/1"] + + +# -- the pure parts ----------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "constraint, bounds", + [ + (">= 0 and <= 100", (Fraction(0), Fraction(100))), + ("between -1.5 and 2.25", (Fraction(-3, 2), Fraction(9, 4))), + ("> 0", (Fraction(0), None)), + ("< 10", (None, Fraction(10))), + ("about 5", (None, None)), + ], +) +def test_a_declared_range_is_read_exactly(constraint, bounds): + column = {"validationRules": [{"type": "range", "constraint": constraint}]} + assert defaults.declared_range(column) == bounds + + +def test_several_range_rules_keep_the_tightest_bounds(): + column = { + "validationRules": [ + {"type": "range", "constraint": ">= 0"}, + {"type": "range", "constraint": ">= 5 and <= 50"}, + {"type": "regex", "constraint": ">= 1000"}, + ] + } + assert defaults.declared_range(column) == (Fraction(5), Fraction(50)) + + +@pytest.mark.parametrize( + "low, high, bits, edges", + [ + (0, 100, 16, [Fraction(n) for n in range(0, 101, 10)]), + (Fraction(1, 2), Fraction(5, 2), 16, [Fraction(n, 4) for n in range(2, 11)]), + (0, 1, 16, [Fraction(n, 8) for n in range(9)]), + (3, 97, 16, [Fraction(3), *(Fraction(n) for n in range(10, 91, 10)), Fraction(97)]), + (0, 1, 0, [Fraction(0), Fraction(1)]), # no step below the grid's + ], +) +def test_a_declared_range_is_cut_into_about_ten_bins_on_the_grid(low, high, bits, edges): + assert list(defaults.range_bins(Fraction(low), Fraction(high), bits)) == edges + + +@pytest.mark.parametrize( + "low, high, edges", + [ + (1, 5, [1, 2, 3, 4, 5, Fraction(11, 2)]), # one bin per value, 5 not beside 4 + (0, 100, [*range(0, 101, 10), Fraction(201, 2)]), + (-3, 3, [-3, -2, -1, 0, 1, 2, 3, Fraction(7, 2)]), + ], +) +def test_an_integral_range_gets_whole_bins_and_half_a_unit_past_its_top(low, high, edges): + got = defaults.range_bins(Fraction(low), Fraction(high), 16, integral=True) + assert list(got) == [Fraction(edge) for edge in edges] + + +def test_on_a_grid_of_whole_steps_an_integral_range_ends_a_whole_unit_past_its_top(): + # Half a unit is off that grid; a whole one keeps 5 out of 4's bin. + got = defaults.range_bins(Fraction(1), Fraction(5), 0, integral=True) + assert list(got) == [Fraction(edge) for edge in (1, 2, 3, 4, 5, 6)] + + +def test_a_range_off_the_grid_is_rounded_outward_to_it(): + got = defaults.range_bins(Fraction("0.01"), Fraction("0.99"), 16) + assert got[0] <= Fraction("0.01") < got[1] and got[-2] < Fraction("0.99") <= got[-1] + assert all((edge * 2**16).denominator == 1 for edge in got) + + +def test_the_number_ladder_steps_1_2_5_and_halves_below_one_for_a_decimal(): + ladder = defaults.number_ladder(Fraction(-10), Fraction(10), sub_unit=True, fractional_bits=3) + assert ladder == tuple( + Fraction(n) + for n in ("-10", "-5", "-2", "-1", "-1/2", "-1/4", "-1/8", "0", "1/8", "1/4", "1/2") + ) + tuple(Fraction(n) for n in ("1", "2", "5", "10")) diff --git a/tests/profiler/test_disclosure_oracle.py b/tests/profiler/test_disclosure_oracle.py new file mode 100644 index 00000000..445bf4cf --- /dev/null +++ b/tests/profiler/test_disclosure_oracle.py @@ -0,0 +1,255 @@ +# Copyright 2025 Agentics Transformation Limited +# SPDX-License-Identifier: Apache-2.0 + +"""An exhaustive audit of the count tables a profile publishes: no small count is pinned. + +For tiny seeded instances (one or two columns, at most three bins, at most seven rows) +this enumerates every world, every multiset of records, runs the real profiler on each +(the registry, the output checks and the document, through ``profile_exposes``), and +groups the worlds by the document they publish. Worlds that publish the same document +are the worlds a reader cannot tell apart, the publish decision included: a withheld +cell is the same ``C`` in each. A count is *pinned* when every world of its group gives +it one value. The audit fails on a count the rule protects (1 to ``minimumCell - 1`` +records, or none where zeros are disclosive) that the new statistics pin and the +document without them does not: a count of one cell, or of any run of adjacent cells, +of each table, so the three cells of a ``quantile/2`` are among them. + +A pin the old document already gave is not this change's, and is not counted against +it: a published ``min`` shows that no record lies below it (PROFILE-SPEC §3, outside +ADR 0012's rule). The rule is scaled down (a minimum cell of 2 to 4, rounding to 1 or 2) +so that seven records can publish anything, with the same structure as ACRO's defaults. +A group of worlds is judged only when the universe holds all of them: every world of it +has at most ``MAX_ROWS - rounding_base`` records, so no larger world is missing from it. + +What it found, and what the profiler does about it: + +- a column's ``distinctCount`` says how many of a histogram's withheld cells hold no + record, so a column a level histogram counts withholds it; +- a number column's exact ``sum`` (and ``mean/1``, its sum on the grid), its + ``quantiles`` and its ``distinctCount`` give a withheld bin back from the published + ones, so a number column the contract's block bins withholds them and forms no + ``mean/1``; +- a histogram and quantiles on the same bins, checked together, let a reader who knows + the check's choices tell worlds apart, so a column gets one or the other. +""" + +from __future__ import annotations + +import itertools +from collections import defaultdict +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple + +import pytest + +from fluid_build.profiler import disclosure, runner, sources +from fluid_build.profiler._stats import load_backend + +pytestmark = pytest.mark.unit + +pytest.importorskip("flux_stats.fidelity") + +#: The most records a world holds. +MAX_ROWS = 7 + +RULES = [ + disclosure.DisclosureRule( + name=f"oracle-m{cell}-r{base}", + minimum_cell=cell, + zeros_disclosive=True, + rounding_base=base, + dominance_n=2, + dominance_k="0.90", + p_ratio="0.10", + ) + for cell, base in ((2, 1), (3, 1), (3, 2), (4, 2)) +] + +Row = Tuple[Optional[str], ...] +#: A protected count: its name, and the function that counts it in a world. +Quantity = Tuple[str, Callable[[Sequence[Row]], int]] + + +def _contract(tables: Mapping[str, Sequence[Tuple[str, str, Mapping[str, Any]]]], dims=None): + exposes = [] + for name, columns in tables.items(): + expose: Dict[str, Any] = { + "exposeId": name, + "contract": {"schema": [{"name": c, "type": t, **extra} for c, t, extra in columns]}, + } + if dims and name in dims: + expose["semantics"] = { + "dimensions": [{"name": d, "type": "categorical"} for d in dims[name]] + } + exposes.append(expose) + return {"fluidVersion": "0.7.7", "kind": "DataProduct", "id": "oracle", "exposes": exposes} + + +def _worlds(values: Sequence[Row], max_rows: int) -> List[Tuple[Row, ...]]: + """Every multiset of at most *max_rows* records drawn from *values*.""" + worlds: List[Tuple[Row, ...]] = [] + for size in range(max_rows + 1): + worlds.extend(itertools.combinations_with_replacement(values, size)) + return worlds + + +def _spans(name: str, cells: Sequence[Callable[[Row], bool]]) -> List[Quantity]: + """The count of each run of adjacent cells, for cells given as membership tests.""" + found: List[Quantity] = [] + for low in range(len(cells)): + for high in range(low + 1, len(cells) + 1): + members = cells[low:high] + + def count(world: Sequence[Row], members=members) -> int: + return sum(1 for row in world if any(test(row) for test in members)) + + found.append((f"{name}[{low}:{high}]", count)) + return found + + +def _without_new_statistics(document: Mapping[str, Any]) -> str: + """The document as it was before the contract's default statistics: mean/1 alone.""" + import json + + stripped = json.loads(json.dumps(document)) + stripped.pop("_schemaURL", None) + for table in stripped["tables"].values(): + for column in table["columns"].values(): + kept = [s for s in column.get("statistics", []) if s["definition"] == "mean/1"] + if kept: + column["statistics"] = kept + else: + column.pop("statistics", None) + return json.dumps(stripped, sort_keys=True) + + +def _audit( + monkeypatch: pytest.MonkeyPatch, + contract: Mapping[str, Any], + columns: Sequence[str], + values: Sequence[Row], + quantities: Sequence[Quantity], + rule: disclosure.DisclosureRule, + max_rows: int = MAX_ROWS, +) -> Tuple[int, List[str]]: + """Profile every world; return how many it ran and each count the change pins.""" + backend = load_backend() + (expose,) = [e["exposeId"] for e in contract["exposes"]] + current: Dict[str, Tuple[Row, ...]] = {} + + def read_columns(source, names, *, table=None, cwd=None): + world = current["world"] + return sources.SourceColumns( + description="world", + row_count=len(world), + values={name: [row[columns.index(name)] for row in world] for name in names}, + ) + + monkeypatch.setattr(sources, "read_columns", read_columns) + new_groups: Dict[bytes, List[Tuple[Row, ...]]] = defaultdict(list) + old_view: Dict[Tuple[Row, ...], str] = {} + old_groups: Dict[str, List[Tuple[Row, ...]]] = defaultdict(list) + worlds = _worlds(values, max_rows) + for world in worlds: + current["world"] = world + result = runner.profile_exposes( + contract, [runner.ExposeSource(expose, "world")], backend=backend, rule=rule + ) + new_groups[result.canonical].append(world) + old_view[world] = _without_new_statistics(result.document) + old_groups[old_view[world]].append(world) + complete = max_rows - rule.rounding_base + pins: List[str] = [] + for group in new_groups.values(): + if any(len(world) > complete for world in group): + continue # a larger world, outside the universe, may publish this document too + for name, count in quantities: + seen = {count(world) for world in group} + if len(seen) != 1: + continue + (value,) = seen + if not (value < rule.minimum_cell and (value > 0 or rule.zeros_disclosive)): + continue # not a count the rule protects + before = {count(world) for world in old_groups[old_view[group[0]]]} + if len(before) > 1: + pins.append(f"{rule.name}: {name} = {value} in worlds like {group[0]}") + return len(worlds), pins + + +LEVELS = ("a", "b", "c") + + +@pytest.mark.parametrize("rule", RULES, ids=lambda rule: rule.name) +def test_a_level_histogram_pins_no_small_count(monkeypatch, rule): + contract = _contract({"t": [("s", "enum('a', 'b', 'c')", {})]}) + values: List[Row] = [(level,) for level in LEVELS] + [(None,)] + cells = [lambda row, level=level: row[0] == level for level in LEVELS] + ran, pins = _audit(monkeypatch, contract, ["s"], values, _spans("s", cells), rule) + assert ran == 330 + assert pins == [] + + +@pytest.mark.parametrize("rule", RULES, ids=lambda rule: rule.name) +def test_a_histogram_and_its_quantiles_over_a_declared_range_pin_no_small_count(monkeypatch, rule): + # between 0 and 2 on an integer column: bins [0, 1), [1, 2) and [2, 2.5], one per value. + declared = {"validationRules": [{"type": "range", "constraint": "between 0 and 2"}]} + contract = _contract({"t": [("v", "int", declared)]}) + values: List[Row] = [("0",), ("1",), ("2",), (None,)] + cells = [lambda row, value=value: row[0] == value for value in ("0", "1", "2")] + ran, pins = _audit(monkeypatch, contract, ["v"], values, _spans("v", cells), rule) + assert ran == 330 + assert pins == [] + + +@pytest.mark.parametrize("rule", RULES, ids=lambda rule: rule.name) +def test_a_grouped_level_histogram_pins_no_small_count(monkeypatch, rule): + contract = _contract( + {"t": [("s", "enum('a', 'b')", {}), ("g", "enum('x', 'y')", {})]}, dims={"t": ["g"]} + ) + values: List[Row] = [(s, g) for s in ("a", "b") for g in ("x", "y")] + quantities: List[Quantity] = [] + for g in ("x", "y"): + cells = [lambda row, s=s, g=g: row == (s, g) for s in ("a", "b")] + quantities.extend(_spans(f"s|g={g}", cells)) + ran, pins = _audit(monkeypatch, contract, ["s", "g"], values, quantities, rule, max_rows=6) + assert ran == 210 + assert pins == [] + + +@pytest.mark.parametrize("rule", RULES, ids=lambda rule: rule.name) +def test_a_grouped_level_histogram_with_nulls_pins_no_small_count(monkeypatch, rule): + # A record can hold a level and a null key, or neither: those cells are protected too. + # The grouped statistic's sampleSize once banded the records its groups hold, and + # beside the columns' own counts that pinned them, under rules that round or not. + contract = _contract( + {"t": [("s", "enum('a', 'b')", {}), ("g", "enum('x', 'y')", {})]}, dims={"t": ["g"]} + ) + levels, keys = ("a", None), ("x", None) + values: List[Row] = [(s, g) for s in levels for g in keys] + quantities: List[Quantity] = [] + for g in keys: + cells = [lambda row, s=s, g=g: row == (s, g) for s in levels] + quantities.extend(_spans(f"s|g={g}", cells)) + for s in levels: + cells = [lambda row, s=s, g=g: row == (s, g) for g in keys] + quantities.extend(_spans(f"g|s={s}", cells)) + ran, pins = _audit(monkeypatch, contract, ["s", "g"], values, quantities, rule) + assert ran == 330 + assert pins == [] + + +@pytest.mark.parametrize("rule", RULES, ids=lambda rule: rule.name) +def test_quantiles_on_a_types_ladder_pin_no_small_count(monkeypatch, rule): + # No declared range: quantile/2 at the quartiles and the median on the int64 ladder, + # whose bins near these values are [-1, 0), [0, 1), [1, 2), [2, 5) and [5, 10). + contract = _contract({"t": [("v", "int", {})]}) + values: List[Row] = [("0",), ("1",), ("2",), (None,)] + cells = [ + lambda row: row[0] is not None and int(row[0]) < 0, + lambda row: row[0] == "0", + lambda row: row[0] == "1", + lambda row: row[0] == "2", + lambda row: row[0] is not None and int(row[0]) >= 5, + ] + ran, pins = _audit(monkeypatch, contract, ["v"], values, _spans("v", cells), rule) + assert ran == 330 + assert pins == [] diff --git a/tests/profiler/test_flux_stats_backend.py b/tests/profiler/test_flux_stats_backend.py index 030b11bc..3633c3ad 100644 --- a/tests/profiler/test_flux_stats_backend.py +++ b/tests/profiler/test_flux_stats_backend.py @@ -203,7 +203,8 @@ def test_a_sum_longer_than_flux_stats_spells_is_refused_with_the_document(tmp_pa # Four distinct 1000-digit values, ten records each: every value is in bound and # published, and their sum (1002 digits) is past what flux-stats spells. contract = { - "exposes": [{"exposeId": "a", "contract": {"schema": [{"name": "n", "type": "integer"}]}}] + # A decimal with no (p,s) has no bound, so no quantile reads these values first. + "exposes": [{"exposeId": "a", "contract": {"schema": [{"name": "n", "type": "decimal"}]}}] } rows = [{"n": 10**999 * (5 + i % 4)} for i in range(40)] path = tmp_path / "a.ndjson" @@ -268,7 +269,11 @@ def test_the_mean_a_profile_carries_is_the_one_the_check_compares(tmp_path, back profile, _ = load_profile_text(result.canonical.decode("utf-8")) assert profile.fractional_bits == 16 for column in ("amount", "attempts"): - (entry,) = profile.column_statistics("payment", column) + (entry,) = [ + each + for each in profile.column_statistics("payment", column) + if each.identifier == "mean/1" + ] assert entry.identifier == "mean/1" and entry.group_by is None (group,) = entry.groups assert group.published and group.key == "" diff --git a/tests/profiler/test_runner.py b/tests/profiler/test_runner.py index 69d776f5..5eb6d3ac 100644 --- a/tests/profiler/test_runner.py +++ b/tests/profiler/test_runner.py @@ -188,9 +188,11 @@ def test_a_profile_publishes_what_passes_and_withholds_the_rest(tmp_path): # The four nullCounts (three zeros and a 4), and the distinctCount of market and # placed_at (2 values each, which round to 0). - assert result.withheld == {"A": 6} - # rowCount, amount's mean/1, and every published metric. - assert result.published == 1 + 1 + sum( + # The quantiles at 0.25 and 0.75 of amount, and all three of placed_at, have a cell of + # no records below or above their bin, so they are withheld beside the six metrics. + assert result.withheld == {"A": 11} + # rowCount, amount's mean/1 and its median's bin, and every published metric. + assert result.published == 1 + 1 + 1 + sum( 1 for col in table["columns"].values() for m in col["metrics"].values() @@ -207,7 +209,7 @@ def test_a_number_column_carries_mean_1_on_the_documents_grid(tmp_path): columns = result.document["tables"]["orders"]["columns"] assert result.document["encoding"] == {"fractionalBits": 16, "rounding": "floor"} # k is rounded as count is; Sx is the sum of the values on the grid. - assert columns["amount"]["statistics"] == [ + assert [s for s in columns["amount"]["statistics"] if s["definition"] == "mean/1"] == [ { "definition": "mean/1", "sampleSize": {"min": 38, "max": 42}, @@ -215,9 +217,18 @@ def test_a_number_column_carries_mean_1_on_the_documents_grid(tmp_path): "groups": [{"group": "", "k": 40, "Sx": str(620 * 2**16), "status": "F"}], } ] - # A timestamp or a category gets no statistic. - assert [name for name, column in columns.items() if "statistics" in column] == ["amount"] - assert result.statistics == (("amount", "mean/1", None, None),) + # A timestamp gets only the default quantiles, and a string without levels nothing. + definitions = { + name: sorted({s["definition"] for s in column.get("statistics", ())}) + for name, column in columns.items() + } + assert definitions == { + "amount": ["mean/1", "quantile/2"], + "market": [], + "placed_at": ["quantile/2"], + "note": [], + } + assert ("amount", "mean/1", None, None) in result.statistics assert result.fractional_bits == 16 @@ -235,10 +246,10 @@ def test_an_order_statistic_too_few_records_hold_is_withheld(tmp_path): def test_a_sum_one_record_dominates_is_withheld(tmp_path): rows = _rows() - rows[0]["amount"] = "1000000.00" + rows[0]["amount"] = "1234567.00" # not an edge of the ladder its quantiles are binned on result = profile_expose(ORDERS, "orders", source=_slice(tmp_path, rows), backend=FakeBackend()) assert _metrics(result, "amount")["sum"]["status"] == "O" - assert "1000000" not in result.canonical.decode("utf-8") + assert "1234567" not in result.canonical.decode("utf-8") def test_a_small_table_publishes_nothing_but_stays_a_valid_profile(tmp_path): @@ -320,7 +331,11 @@ def test_a_constant_column_withholds_its_sum_and_publishes_its_value(tmp_path): def _mean(result, column="amount"): - (statistic,) = result.document["tables"]["orders"]["columns"][column]["statistics"] + (statistic,) = [ + each + for each in result.document["tables"]["orders"]["columns"][column]["statistics"] + if each["definition"] == "mean/1" + ] assert statistic["definition"] == "mean/1" (group,) = statistic["groups"] return statistic, group @@ -336,7 +351,7 @@ def _amounts(values): @pytest.mark.parametrize( ("values", "status"), [ - (["1000000.00"] + ["1.00", "2.00", "3.00"] * 13, "O"), # one record dominates the sum + (["1234567.00"] + ["1.00", "2.00", "3.00"] * 13, "O"), # one record dominates the sum (["1"] * 3 + ["0"] * 37, "A"), # 3 records contribute: a sum of 0s and 1s is a count (["7.00"] * 40, "C"), # one value: the sum is a count of it (["1.00", "2.00"] * 20, "C"), # two values, both published as min and max @@ -350,8 +365,8 @@ def test_a_mean_is_withheld_where_the_sum_is(tmp_path, values, status): statistic, group = _mean(result) assert group == {"group": "", "status": status} assert statistic["sampleSize"] == {"min": 38, "max": 42} - assert result.statistics == () - assert "1000000" not in result.canonical.decode("utf-8") + assert not [each for each in result.statistics if each[1] == "mean/1"] + assert "1234567" not in result.canonical.decode("utf-8") # no edge of the ladder either def test_a_small_table_withholds_its_mean_as_a_small_count(tmp_path): @@ -437,7 +452,8 @@ def test_a_column_with_no_value_has_no_mean_where_zeros_publish(tmp_path): published = profile_expose( ORDERS, "orders", source=source, backend=FakeBackend(), rule=zeros_publish ) - assert "statistics" not in published.document["tables"]["orders"]["columns"]["amount"] + statistics = published.document["tables"]["orders"]["columns"]["amount"]["statistics"] + assert "mean/1" not in {each["definition"] for each in statistics} @pytest.mark.parametrize("fractional_bits", [-1, 129, True, "16"]) @@ -470,3 +486,121 @@ def test_prices_that_all_end_in_99_withhold_the_mean_but_not_the_sum(tmp_path): ) assert _metrics(result, "amount")["sum"]["status"] == "F" assert _mean(result)[1] == {"group": "", "status": "C"} + + +# -- several exposes, one document ----------------------------------------------------- + + +def _two_tables(): + return { + "fluidVersion": "0.7.7", + "kind": "DataProduct", + "id": "p", + "exposes": [ + {"exposeId": "org", "contract": {"schema": [{"name": "name", "type": "string"}]}}, + { + "exposeId": "account", + "contract": {"schema": [{"name": "plan", "type": "string"}]}, + }, + ], + } + + +def _ndjson(path: Path, rows) -> str: + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return str(path) + + +def test_several_exposes_go_into_one_document_each_from_its_own_source(tmp_path): + org = _ndjson(tmp_path / "org.ndjson", [{"name": f"o{n % 3}"} for n in range(12)]) + account = _ndjson(tmp_path / "acct.ndjson", [{"plan": "basic"} for _ in range(20)]) + result = runner.profile_exposes( + _two_tables(), + [runner.ExposeSource("account", account), runner.ExposeSource("org", org)], + backend=FakeBackend(), + ) + tables = result.document["tables"] + assert list(tables) == ["account", "org"] + assert tables["org"]["rowCount"] == 10 and tables["account"]["rowCount"] == 20 + assert [table.plan.expose_id for table in result.tables] == ["account", "org"] + assert result.digest == digest_of(result.canonical) + with pytest.raises(ValueError, match="one result per expose"): + result.plan # noqa: B018 - a profile of two exposes has no one plan + + +def test_an_expose_named_twice_is_refused_before_any_read(tmp_path): + source = _ndjson(tmp_path / "org.ndjson", [{"name": "x"}]) + with pytest.raises(ProfileError) as refused: + runner.profile_exposes( + _two_tables(), + [runner.ExposeSource("org", source), runner.ExposeSource("org", "missing.ndjson")], + backend=FakeBackend(), + ) + assert refused.value.event == "profile_expose_repeated" + + +def test_one_expose_profiles_as_it_always_did(tmp_path): + rows = _rows() + one = profile_expose(ORDERS, "orders", source=_slice(tmp_path, rows), backend=FakeBackend()) + many = runner.profile_exposes( + ORDERS, [runner.ExposeSource("orders", _slice(tmp_path, rows))], backend=FakeBackend() + ) + assert one.canonical == many.canonical + assert one.plan == many.plan + + +# -- a column the contract types date ----------------------------------------------------- + +_DATED = _contract(("signed_on", "date"), ("seen_at", "timestamp"), expose_id="contracts") + + +def test_a_date_written_yyyy_mm_dd_is_read_as_that_day(tmp_path): + rows = [{"signed_on": "2026-01-31", "seen_at": "2026-01-31T00:00:00.000Z"} for _ in range(12)] + result = profile_expose( + _DATED, "contracts", source=_ndjson(tmp_path / "c.ndjson", rows), backend=FakeBackend() + ) + columns = result.document["tables"]["contracts"]["columns"] + # The day's midnight in UTC, in milliseconds since the epoch, as a Parquet date reads. + assert columns["signed_on"]["metrics"]["min"]["value"] == "1769817600000" + assert columns["signed_on"]["metrics"] == columns["seen_at"]["metrics"] + + +@pytest.mark.parametrize("text", ["2026-02-30", "31/01/2026", "2026-1-31", "2026-01-31 10:00"]) +def test_a_date_not_written_as_a_calendar_day_is_refused(tmp_path, text): + rows = [{"signed_on": text, "seen_at": None}] + with pytest.raises(ReadRefused, match=r"column 'signed_on' row 0") as refused: + profile_expose( + _DATED, "contracts", source=_ndjson(tmp_path / "c.ndjson", rows), backend=FakeBackend() + ) + assert text not in str(refused.value) + + +def test_a_timestamp_column_still_needs_rfc_3339(tmp_path): + rows = [{"signed_on": None, "seen_at": "2026-01-31"}] + with pytest.raises(ReadRefused, match=r"column 'seen_at' row 0"): + profile_expose( + _DATED, "contracts", source=_ndjson(tmp_path / "c.ndjson", rows), backend=FakeBackend() + ) + + +def test_a_csv_date_column_profiles_as_the_same_parquet_column_does(tmp_path): + duckdb = pytest.importorskip("duckdb") + backend = load_backend() + days = [f"2026-0{1 + n % 3}-{10 + n % 7:02d}" for n in range(60)] + csv = tmp_path / "c.csv" + csv.write_text("signed_on,seen_at\n" + "".join(f"{d},\n" for d in days), encoding="utf-8") + parquet = tmp_path / "c.parquet" + con = duckdb.connect() + try: + con.execute( + "COPY (SELECT CAST(signed_on AS DATE) AS signed_on, " + "CAST(NULL AS TIMESTAMPTZ) AS seen_at " + f"FROM read_csv('{csv}', all_varchar = true)) TO '{parquet}' (FORMAT parquet)" + ) + finally: + con.close() + from_csv = profile_expose(_DATED, "contracts", source=str(csv), backend=backend) + from_parquet = profile_expose(_DATED, "contracts", source=str(parquet), backend=backend) + signed = from_csv.document["tables"]["contracts"]["columns"]["signed_on"] + assert signed["domain"] == "timestamp" + assert signed == from_parquet.document["tables"]["contracts"]["columns"]["signed_on"] diff --git a/tests/profiler/test_sources.py b/tests/profiler/test_sources.py index 5234ff40..a0589f48 100644 --- a/tests/profiler/test_sources.py +++ b/tests/profiler/test_sources.py @@ -218,6 +218,16 @@ def test_a_database_uri_is_shown_with_its_password_masked(): assert SECRET not in shown and "reader" in shown +@pytest.mark.parametrize("name", ["orders", "ordres"]) # an exposeId, or a mistyped one +def test_a_database_uri_after_a_name_is_masked_too(name): + shown = describe_source(f"{name}=postgresql://reader:{SECRET}@db.internal/sales?ssl=on") + assert SECRET not in shown and shown.startswith(f"{name}=postgresql://reader") + + +def test_a_path_holding_an_equals_sign_is_shown_as_given(): + assert describe_source("./orders=2024.csv") == "./orders=2024.csv" + + def test_a_connect_failure_masks_the_password(): with pytest.raises(SourceError) as raised: read_columns( diff --git a/tests/providers/test_quality_engine_column_rules.py b/tests/providers/test_quality_engine_column_rules.py index cf94a412..32c37d9b 100644 --- a/tests/providers/test_quality_engine_column_rules.py +++ b/tests/providers/test_quality_engine_column_rules.py @@ -358,6 +358,7 @@ def test_non_string_value_rejected(): ValueDomain, execute_quality_checks, parse_range, + parse_range_exact, value_domain, ) @@ -378,6 +379,21 @@ def test_parse_range(constraint, bounds): assert parse_range(constraint) == bounds +def test_parse_range_exact_reads_the_same_grammar_without_rounding(): + from fractions import Fraction + + assert parse_range_exact("between -1.5 and 0.1") == [ + (">=", Fraction(-3, 2)), + ("<=", Fraction(1, 10)), # exactly one tenth, which no float holds + ] + for constraint in (">= 0", ">= 0 and <= 100", "> 0 AND < 1", "0..100", ">= zero", None): + exact = parse_range_exact(constraint) + assert [(op, float(value)) for op, value in exact] == parse_range(constraint) + # A bound past what anyone writes is no bound, as parse_range reads an infinite one. + assert parse_range_exact(">= " + "9" * 1001) == [] + assert parse_range(">= " + "9" * 400) == [] + + ENERGY = [ {"name": "grid_kwh", "validationRules": [{"type": "range", "constraint": ">= 0"}]}, {"name": "solar_kwh", "validationRules": [{"type": "range", "constraint": ">= 0"}]},