From 379d0c1b4641d13008df2af0a0e06090252f8fd8 Mon Sep 17 00:00:00 2001 From: Vardhman Gupta Date: Tue, 3 Nov 2026 14:52:44 +0530 Subject: [PATCH] feat(eval): add regression runs and CI gating (#305) Introduce regression checking, CI gating, and cross-version history persistence for dynaevals. - Add check_regression() and assert_no_regression() in src/dynavec/eval/gate.py - Support metric polarity (higher-is-better vs lower-is-better) and per-metric threshold overrides - Add dynavec eval check CLI subcommand with exit code 1 on regression - Add save_json() and load_json() to EvalRunStore with atomic writes - Document GitHub Actions workflow example in docs/eval_ci_gating.md - Add comprehensive unit tests in tests/test_eval_gate.py, test_cli.py, and test_eval_trend.py --- docs/eval_ci_gating.md | 165 ++++++++++++++ src/dynavec/cli.py | 143 +++++++++++++ src/dynavec/eval/__init__.py | 15 +- src/dynavec/eval/gate.py | 403 +++++++++++++++++++++++++++++++++++ src/dynavec/eval/trend.py | 87 +++++++- tests/test_cli.py | 87 ++++++++ tests/test_eval_gate.py | 282 ++++++++++++++++++++++++ tests/test_eval_trend.py | 78 +++++-- 8 files changed, 1244 insertions(+), 16 deletions(-) create mode 100644 docs/eval_ci_gating.md create mode 100644 src/dynavec/eval/gate.py create mode 100644 tests/test_eval_gate.py diff --git a/docs/eval_ci_gating.md b/docs/eval_ci_gating.md new file mode 100644 index 0000000..f7130a6 --- /dev/null +++ b/docs/eval_ci_gating.md @@ -0,0 +1,165 @@ +# Evaluation Regression Runs & CI Gating + +`dynaevals` includes built-in regression detection and CI gating to prevent performance drops (retrieval quality, faithfulness, relevance, or latency regressions) from slipping into production. + +--- + +## 1. Overview + +When developing agent workflows, RAG pipelines, or retrieval systems, code changes, chunking adjustments, or prompt changes can silently degrade search accuracy or answer quality. + +Regression gating solves this by comparing an **evaluation run** against an established **baseline**: +- **Automatic Polarity**: Quality metrics (`recall@k`, `mrr`, `ndcg@k`, `faithfulness`, `relevance`) must not decrease below a threshold. Latency metrics (`p50_latency_ms`, `p95_latency_ms`) must not increase beyond a threshold. +- **CI Gate**: Non-zero exit code (`1`) immediately fails CI builds when any gated metric regresses. +- **Trend History**: Persists evaluation snapshots across versions/commits in `EvalRunStore`. + +--- + +## 2. CLI Usage (`dynavec eval check`) + +The `dynavec eval check` command compares a current eval result against a baseline JSON snapshot. + +```bash +dynavec eval check \ + --current current_eval.json \ + --baseline baseline_eval.json \ + --max-drop 0.05 +``` + +### CLI Arguments + +| Flag | Description | Default | +|------|-------------|---------| +| `--current`, `-c` | Path to current run eval summary JSON | **Required** | +| `--baseline`, `-b` | Path to baseline run eval summary JSON | **Required** (unless `--allow-missing-baseline`) | +| `--max-drop` | Max allowed relative degradation | `0.05` (5%) | +| `--threshold METRIC=VAL` | Override threshold for a specific metric (repeatable) | None | +| `--absolute` | Compare absolute drop instead of relative change | `False` | +| `--store FILE` | Path to `EvalRunStore` JSON file to persist history | None | +| `--tag KEY=VALUE` | Attach metadata tag to the stored run (repeatable) | None | +| `--json` | Emit structured JSON report instead of text table | `False` | +| `--allow-missing-baseline` | Succeed with warning if baseline file does not exist | `False` | + +### Exit Codes + +- `0`: All metrics passed (within thresholds or improved). +- `1`: One or more metrics regressed beyond threshold (CI fails). +- `2`: Missing file or invalid CLI arguments. + +### Output Example + +```text +Metric | Baseline | Current | Delta | % Change | Threshold | Status +---------------+----------+---------+---------+----------+-----------+----------- +mrr | 0.8200 | 0.8350 | +0.0150 | +1.83% | <= 5.0% | IMPROVED +p95_latency_ms | 32.0000 | 33.5000 | +1.5000 | +4.69% | <= 5.0% | PASS +recall@10 | 0.9100 | 0.8200 | -0.0900 | -9.89% | <= 5.0% | REGRESSION + +Overall: FAILED: 1 metric(s) regressed beyond threshold. +``` + +--- + +## 3. GitHub Actions Workflow Example + +Create `.github/workflows/eval-gate.yml` in your repository: + +```yaml +name: Evaluation Regression Gate + +on: + pull_request: + branches: [main, development] + push: + branches: [main] + +jobs: + eval-gate: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + pip install --upgrade pip + pip install -e ".[dev]" + + - name: Cache eval baseline + id: cache-baseline + uses: actions/cache@v4 + with: + path: .eval_cache/baseline.json + key: eval-baseline-${{ runner.os }} + + - name: Run benchmark evaluation + run: | + python -m tests.run_eval_benchmark --output current_eval.json + + - name: Gate on regressions + run: | + dynavec eval check \ + --current current_eval.json \ + --baseline .eval_cache/baseline.json \ + --max-drop 0.05 \ + --threshold p95_latency_ms=0.15 \ + --allow-missing-baseline \ + --store .eval_cache/eval_history.json \ + --tag commit=${{ github.sha }} \ + --tag branch=${{ github.ref_name }} + + - name: Update baseline on main branch + if: github.ref == 'refs/heads/main' && success() + run: | + mkdir -p .eval_cache + cp current_eval.json .eval_cache/baseline.json +``` + +--- + +## 4. Python Testing API (`assert_no_regression`) + +You can also enforce regression gates inside `pytest` test suites: + +```python +from dynavec.eval import assert_no_regression, EvalRunner, MockJudge + +def test_rag_quality_no_regression(): + runner = EvalRunner(judge=MockJudge()) + current_summary = runner.run(test_dataset) + + # Compare against saved baseline + assert_no_regression( + current=current_summary, + baseline="benchmarks/baselines/v0.7.0.json", + max_drop=0.05, + metric_thresholds={"faithfulness": 0.03}, + ) +``` + +--- + +## 5. Cross-Version Trend Persistence (`EvalRunStore`) + +`EvalRunStore` can persist evaluation history to disk and reload it: + +```python +from dynavec.eval import EvalRunStore + +# Load existing history (or create new if file doesn't exist) +store = EvalRunStore.load_json("eval_history.json") + +# Record new evaluation run +store.record(summary, tags={"version": "0.7.1", "git_commit": "abc1234"}) + +# Persist to disk atomically +store.save_json("eval_history.json") + +# Query trends over time +recall_trend = store.trend("recall", k=10) # [(timestamp, value), ...] +``` diff --git a/src/dynavec/cli.py b/src/dynavec/cli.py index 817b6d6..f2d8b28 100644 --- a/src/dynavec/cli.py +++ b/src/dynavec/cli.py @@ -86,6 +86,51 @@ def _parser() -> argparse.ArgumentParser: default=8000, help="Port for SSE transport (default: 8000)", ) + + eval_cmd = subparsers.add_parser("eval", help="evaluation regression gating and trend commands") + eval_sub = eval_cmd.add_subparsers(dest="eval_command") + eval_check = eval_sub.add_parser( + "check", help="compare eval run against baseline and fail on regression" + ) + eval_check.add_argument("--current", "-c", required=True, help="path to current eval run JSON file") + eval_check.add_argument( + "--baseline", "-b", required=False, help="path to baseline eval run JSON file" + ) + eval_check.add_argument( + "--max-drop", + type=float, + default=0.05, + help="max allowed relative drop (default: 0.05)", + ) + eval_check.add_argument( + "--threshold", + action="append", + dest="thresholds", + metavar="METRIC=VAL", + help="per-metric threshold override, e.g. --threshold recall@10=0.02 (repeatable)", + ) + eval_check.add_argument( + "--absolute", + action="store_true", + help="use absolute drop threshold instead of relative", + ) + eval_check.add_argument( + "--store", + help="path to EvalRunStore JSON file to record/persist runs", + ) + eval_check.add_argument( + "--tag", + action="append", + dest="tags", + metavar="KEY=VALUE", + help="metadata tags to attach to the recorded run (repeatable)", + ) + eval_check.add_argument("--json", action="store_true", help="output report as JSON") + eval_check.add_argument( + "--allow-missing-baseline", + action="store_true", + help="succeed with warning if baseline file does not exist", + ) return parser @@ -280,6 +325,99 @@ def _check_dynamodb(session: Any, table: str, region: str | None) -> str: return "Accessible" +def _eval_check(args: argparse.Namespace) -> int: + import json + from pathlib import Path + + from .eval.gate import check_regression + from .eval.trend import EvalRun, EvalRunStore + + current_path = Path(args.current) + if not current_path.exists(): + print(f"[FAIL] Current eval file not found: {args.current}", file=sys.stderr) + return 2 + + tag_dict: dict[str, str] = {} + if args.tags: + for item in args.tags: + if "=" in item: + k, v = item.split("=", 1) + tag_dict[k.strip()] = v.strip() + + if args.store: + try: + import time + import uuid + + from .eval.gate import _extract_metrics + + store = EvalRunStore.load_json(args.store) + curr_metrics, c_run_id = _extract_metrics(current_path) + run_id = c_run_id or uuid.uuid4().hex[:12] + run = EvalRun( + run_id=run_id, + ts=time.time(), + tags=tag_dict, + metrics=curr_metrics, + ) + store._runs.append(run) + store.save_json(args.store) + except Exception as exc: + print(f"[WARN] Failed to update trend store at {args.store}: {exc}", file=sys.stderr) + + if not args.baseline: + if args.allow_missing_baseline: + print( + "[WARN] No baseline provided and --allow-missing-baseline set. Skipping regression check." + ) + return 0 + print( + "[FAIL] Missing required --baseline argument (or pass --allow-missing-baseline).", + file=sys.stderr, + ) + return 2 + + baseline_path = Path(args.baseline) + if not baseline_path.exists(): + if args.allow_missing_baseline: + print( + f"[WARN] Baseline file not found: {args.baseline}. Bootstrapping mode: skipping regression check." + ) + return 0 + print(f"[FAIL] Baseline file not found: {args.baseline}", file=sys.stderr) + return 2 + + metric_thresholds: dict[str, float] = {} + if args.thresholds: + for item in args.thresholds: + if "=" in item: + m, val = item.split("=", 1) + try: + metric_thresholds[m.strip()] = float(val.strip()) + except ValueError: + print(f"[FAIL] Invalid threshold value for {m}: {val}", file=sys.stderr) + return 2 + + try: + report = check_regression( + current=current_path, + baseline=baseline_path, + max_drop=args.max_drop, + metric_thresholds=metric_thresholds, + relative=not args.absolute, + ) + except Exception as exc: + print(f"[FAIL] Regression check error: {exc}", file=sys.stderr) + return 2 + + if args.json: + print(json.dumps(report.to_dict(), indent=2)) + else: + print(report.summary_table()) + + return 0 if report.passed else 1 + + def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) if args.command == "version": @@ -301,6 +439,11 @@ def main(argv: list[str] | None = None) -> int: else: server.run(transport="stdio") return 0 + if args.command == "eval": + if getattr(args, "eval_command", None) == "check": + return _eval_check(args) + _parser().parse_args(["eval", "--help"]) + return 0 _parser().print_help() return 0 diff --git a/src/dynavec/eval/__init__.py b/src/dynavec/eval/__init__.py index 700d796..345fa84 100644 --- a/src/dynavec/eval/__init__.py +++ b/src/dynavec/eval/__init__.py @@ -1,4 +1,4 @@ -"""Evaluation framework for RAG generation quality, context quality, and retrieval metrics.""" +"""Evaluation framework for RAG generation quality, context quality, retrieval metrics, and CI regression gating.""" from __future__ import annotations @@ -14,6 +14,13 @@ extract_json, ) from .chart import plot_eval_summary, plot_retrieval_metrics +from .gate import ( + MetricDiff, + MetricDirection, + RegressionReport, + assert_no_regression, + check_regression, +) from .judges import ( BedrockJudge, ChatModelJudge, @@ -48,6 +55,12 @@ __all__ = [ "EvalRun", "EvalRunStore", + # CI Regression Gating + "MetricDirection", + "MetricDiff", + "RegressionReport", + "check_regression", + "assert_no_regression", # LLM Judge & Data Models "BaseJudge", "OpenAIJudge", diff --git a/src/dynavec/eval/gate.py b/src/dynavec/eval/gate.py new file mode 100644 index 0000000..e9b27d9 --- /dev/null +++ b/src/dynavec/eval/gate.py @@ -0,0 +1,403 @@ +"""CI regression gating and run comparison for evaluation metrics.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Any + +from .retrieval import RetrievalEvalSummary +from .runner import EvalSummary +from .trend import EvalRun, _metrics_from_rag, _metrics_from_retrieval + + +class MetricDirection(str, Enum): + """Directionality of metric quality.""" + + HIGHER_IS_BETTER = "higher_is_better" + LOWER_IS_BETTER = "lower_is_better" + + +# Known lower-is-better metrics (where an increase indicates degradation/regression) +LOWER_IS_BETTER_METRICS = { + "p50_latency_ms", + "p95_latency_ms", + "p99_latency_ms", + "mean_latency_ms", + "latency", + "latency_ms", + "error_rate", + "token_count", + "cost", +} + + +def get_metric_direction( + metric_name: str, + overrides: dict[str, MetricDirection] | None = None, +) -> MetricDirection: + """Return whether higher or lower values indicate better quality for a metric.""" + if overrides and metric_name in overrides: + return overrides[metric_name] + name_lower = metric_name.lower() + if ( + name_lower in LOWER_IS_BETTER_METRICS + or "latency" in name_lower + or "duration" in name_lower + or "error_rate" in name_lower + or "cost" in name_lower + ): + return MetricDirection.LOWER_IS_BETTER + return MetricDirection.HIGHER_IS_BETTER + + +@dataclass +class MetricDiff: + """Comparison of a single metric between baseline and current runs.""" + + metric: str + baseline: float + current: float + delta: float + relative_change: float + threshold: float + direction: MetricDirection + is_regression: bool + status: str # "PASS", "REGRESSION", "IMPROVED" + + def to_dict(self) -> dict[str, Any]: + return { + "metric": self.metric, + "baseline": self.baseline, + "current": self.current, + "delta": round(self.delta, 6) if not math.isnan(self.delta) else None, + "relative_change": round(self.relative_change, 6) if not math.isnan(self.relative_change) else None, + "threshold": self.threshold, + "direction": self.direction.value, + "is_regression": self.is_regression, + "status": self.status, + } + + +@dataclass +class RegressionReport: + """Aggregated regression analysis between two eval runs.""" + + passed: bool + diffs: dict[str, MetricDiff] + regressions: list[MetricDiff] + baseline_id: str | None = None + current_id: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "passed": self.passed, + "baseline_id": self.baseline_id, + "current_id": self.current_id, + "regressions_count": len(self.regressions), + "regressions": [r.to_dict() for r in self.regressions], + "diffs": {k: v.to_dict() for k, v in self.diffs.items()}, + } + + def summary_table(self) -> str: + """Render a formatted ASCII/Markdown table of the regression comparison.""" + if not self.diffs: + verdict = "PASSED: Zero metrics compared (empty runs)." if self.passed else ( + "FAILED: No common metrics found between baseline and current runs." + ) + return f"No common metrics evaluated.\n\nOverall: {verdict}" + + headers = ["Metric", "Baseline", "Current", "Delta", "% Change", "Threshold", "Status"] + rows: list[list[str]] = [] + + for name, diff in sorted(self.diffs.items()): + b_str = f"{diff.baseline:.4f}" if not math.isnan(diff.baseline) else "NaN" + c_str = f"{diff.current:.4f}" if not math.isnan(diff.current) else "NaN" + d_str = f"{diff.delta:+.4f}" if not math.isnan(diff.delta) else "NaN" + pct = f"{diff.relative_change * 100:+.2f}%" if not math.isnan(diff.relative_change) else "N/A" + rows.append([ + name, + b_str, + c_str, + d_str, + pct, + f"<= {diff.threshold * 100:.1f}%" if diff.threshold else "0.0%", + diff.status, + ]) + + # Compute column widths + col_widths = [len(h) for h in headers] + for row in rows: + for i, cell in enumerate(row): + col_widths[i] = max(col_widths[i], len(cell)) + + def fmt_row(items: list[str]) -> str: + return " | ".join(item.ljust(col_widths[i]) for i, item in enumerate(items)) + + sep = "-+-".join("-" * w for w in col_widths) + table_lines = [ + fmt_row(headers), + sep, + *(fmt_row(r) for r in rows), + ] + + verdict = "PASSED: No metric regressions detected." if self.passed else ( + f"FAILED: {len(self.regressions)} metric(s) regressed beyond threshold." + ) + return "\n".join(table_lines) + f"\n\nOverall: {verdict}" + + +def _extract_metrics(run_or_summary: Any) -> tuple[dict[str, float], str | None]: + """Extract a flat metric dict and optional run_id from various summary representations.""" + if isinstance(run_or_summary, (str, Path)): + path = Path(run_or_summary).resolve() + if not path.exists(): + raise FileNotFoundError(f"Eval summary file not found: {path}") + with open(path, encoding="utf-8") as f: + data = json.load(f) + return _extract_metrics(data) + + if isinstance(run_or_summary, RetrievalEvalSummary): + return _metrics_from_retrieval(run_or_summary), None + + if isinstance(run_or_summary, EvalSummary): + return _metrics_from_rag(run_or_summary), None + + if isinstance(run_or_summary, EvalRun): + return dict(run_or_summary.metrics), run_or_summary.run_id + + if isinstance(run_or_summary, dict): + run_id = run_or_summary.get("run_id") + # Check if it's an EvalRun dict + if "metrics" in run_or_summary and isinstance(run_or_summary["metrics"], dict): + return dict(run_or_summary["metrics"]), str(run_id) if run_id else None + + # Check if it's a serialized RetrievalEvalSummary + if "mean_recall" in run_or_summary: + out: dict[str, float] = {} + if "total_queries" in run_or_summary: + out["total_queries"] = float(run_or_summary["total_queries"]) + if "mean_mrr" in run_or_summary: + out["mrr"] = float(run_or_summary["mean_mrr"]) + if "p50_latency_ms" in run_or_summary: + out["p50_latency_ms"] = float(run_or_summary["p50_latency_ms"]) + if "p95_latency_ms" in run_or_summary: + out["p95_latency_ms"] = float(run_or_summary["p95_latency_ms"]) + for k, v in run_or_summary.get("mean_recall", {}).items(): + out[f"recall@{k}"] = float(v) + for k, v in run_or_summary.get("mean_ndcg", {}).items(): + out[f"ndcg@{k}"] = float(v) + for k, v in run_or_summary.get("mean_precision", {}).items(): + out[f"precision@{k}"] = float(v) + return out, str(run_id) if run_id else None + + # Check if it's a serialized EvalSummary + if "mean_faithfulness" in run_or_summary: + out = { + "faithfulness": float(run_or_summary.get("mean_faithfulness", 0.0)), + "relevance": float(run_or_summary.get("mean_relevance", 0.0)), + "pass_rate": float(run_or_summary.get("pass_rate", 0.0)), + "p95_latency_ms": float(run_or_summary.get("p95_latency_ms", 0.0)), + } + if "mean_context_precision" in run_or_summary: + out["context_precision"] = float(run_or_summary["mean_context_precision"]) + if "mean_context_recall" in run_or_summary: + out["context_recall"] = float(run_or_summary["mean_context_recall"]) + return out, str(run_id) if run_id else None + + # Raw flat dict of metrics + clean: dict[str, float] = {} + for k, v in run_or_summary.items(): + if isinstance(v, (int, float)): + clean[k] = float(v) + return clean, str(run_id) if run_id else None + + raise TypeError(f"Cannot extract metrics from type: {type(run_or_summary).__name__}") + + +def check_regression( + current: EvalSummary | RetrievalEvalSummary | EvalRun | dict[str, Any] | str | Path, + baseline: EvalSummary | RetrievalEvalSummary | EvalRun | dict[str, Any] | str | Path, + *, + max_drop: float = 0.05, + metric_thresholds: dict[str, float] | None = None, + relative: bool = True, + direction_overrides: dict[str, MetricDirection] | None = None, + fail_on_missing_metric: bool = False, + allow_empty: bool = False, +) -> RegressionReport: + """Compare a current evaluation run against a baseline run and report metric regressions. + + Parameters + ---------- + current: + Current evaluation run (summary object, dict, or path to JSON file). + baseline: + Baseline evaluation run to compare against. + max_drop: + Default allowed metric degradation (e.g. 0.05 for 5% relative drop or 0.05 absolute). + metric_thresholds: + Optional per-metric threshold overrides, e.g. ``{"recall@10": 0.02, "p95_latency_ms": 0.10}``. + relative: + Whether thresholds represent relative change (default True) or absolute delta. + direction_overrides: + Optional custom directions mapping metric names to ``MetricDirection``. + fail_on_missing_metric: + If True, flags a regression when a metric listed in ``metric_thresholds`` is absent. + allow_empty: + If True, returns pass when zero common metrics are present. Otherwise fails. + + Returns + ------- + RegressionReport containing all metric diffs, list of regressions, and pass/fail boolean. + """ + curr_metrics, curr_id = _extract_metrics(current) + base_metrics, base_id = _extract_metrics(baseline) + + per_metric = metric_thresholds or {} + all_keys = set(curr_metrics.keys()) | set(base_metrics.keys()) + diffs: dict[str, MetricDiff] = {} + regressions: list[MetricDiff] = [] + + epsilon = 1e-9 + + for key in sorted(all_keys): + in_curr = key in curr_metrics + in_base = key in base_metrics + + if not in_curr or not in_base: + if fail_on_missing_metric and key in per_metric: + missing_in = "current" if not in_curr else "baseline" + threshold = per_metric[key] + diff = MetricDiff( + metric=key, + baseline=base_metrics.get(key, float("nan")), + current=curr_metrics.get(key, float("nan")), + delta=float("nan"), + relative_change=float("nan"), + threshold=threshold, + direction=get_metric_direction(key, direction_overrides), + is_regression=True, + status=f"REGRESSION (missing in {missing_in})", + ) + diffs[key] = diff + regressions.append(diff) + continue + + c_val = curr_metrics[key] + b_val = base_metrics[key] + direction = get_metric_direction(key, direction_overrides) + threshold = per_metric.get(key, max_drop) + + # Handle NaN or Inf in metric values + if math.isnan(c_val) or math.isnan(b_val): + diff = MetricDiff( + metric=key, + baseline=b_val, + current=c_val, + delta=float("nan"), + relative_change=float("nan"), + threshold=threshold, + direction=direction, + is_regression=True, + status="REGRESSION (NaN value detected)", + ) + diffs[key] = diff + regressions.append(diff) + continue + + delta = c_val - b_val + if abs(b_val) > epsilon: + rel_change = delta / abs(b_val) + else: + rel_change = 0.0 + + is_regression = False + if direction == MetricDirection.HIGHER_IS_BETTER: + # Drop in score is bad + if relative and abs(b_val) > epsilon: + allowed_min = b_val * (1.0 - threshold) - epsilon + is_regression = c_val < allowed_min + else: + is_regression = (b_val - c_val) > (threshold + epsilon) + + if is_regression: + status = "REGRESSION" + elif c_val > (b_val + epsilon): + status = "IMPROVED" + else: + status = "PASS" + + else: # MetricDirection.LOWER_IS_BETTER + # Increase in latency / error is bad + if relative and abs(b_val) > epsilon: + allowed_max = b_val * (1.0 + threshold) + epsilon + is_regression = c_val > allowed_max + else: + is_regression = (c_val - b_val) > (threshold + epsilon) + + if is_regression: + status = "REGRESSION" + elif c_val < (b_val - epsilon): + status = "IMPROVED" + else: + status = "PASS" + + diff = MetricDiff( + metric=key, + baseline=b_val, + current=c_val, + delta=delta, + relative_change=rel_change, + threshold=threshold, + direction=direction, + is_regression=is_regression, + status=status, + ) + diffs[key] = diff + if is_regression: + regressions.append(diff) + + # If no common metrics were evaluated + if not diffs: + passed = allow_empty + else: + passed = len(regressions) == 0 + + return RegressionReport( + passed=passed, + diffs=diffs, + regressions=regressions, + baseline_id=base_id, + current_id=curr_id, + ) + + +def assert_no_regression( + current: Any, + baseline: Any, + *, + max_drop: float = 0.05, + metric_thresholds: dict[str, float] | None = None, + relative: bool = True, + direction_overrides: dict[str, MetricDirection] | None = None, + allow_empty: bool = False, +) -> None: + """Assert that current eval run does not regress compared to baseline. + + Raises :class:`AssertionError` with a comparison table if any metric regresses. + """ + report = check_regression( + current=current, + baseline=baseline, + max_drop=max_drop, + metric_thresholds=metric_thresholds, + relative=relative, + direction_overrides=direction_overrides, + allow_empty=allow_empty, + ) + if not report.passed: + raise AssertionError(f"Eval metric regression detected:\n{report.summary_table()}") diff --git a/src/dynavec/eval/trend.py b/src/dynavec/eval/trend.py index 5d91ab0..087f36d 100644 --- a/src/dynavec/eval/trend.py +++ b/src/dynavec/eval/trend.py @@ -15,13 +15,21 @@ # compare two runs side by side diff = store.compare([run_a_id, run_b_id]) + + # persist and restore across runs + store.save_json("eval_history.json") + loaded = EvalRunStore.load_json("eval_history.json") """ from __future__ import annotations +import json +import os +import tempfile import time import uuid from dataclasses import dataclass +from pathlib import Path from typing import Any from .retrieval import RetrievalEvalSummary @@ -45,6 +53,16 @@ def to_dict(self) -> dict[str, Any]: "metrics": self.metrics, } + @classmethod + def from_dict(cls, data: dict[str, Any]) -> EvalRun: + if not isinstance(data, dict): + raise TypeError(f"Expected dict for EvalRun, got {type(data).__name__}") + run_id = str(data.get("run_id", "")) + ts = float(data.get("ts", 0.0)) + tags = {str(k): str(v) for k, v in data.get("tags", {}).items()} + metrics = dict(data.get("metrics", {})) + return cls(run_id=run_id, ts=ts, tags=tags, metrics=metrics) + def _metrics_from_retrieval(summary: RetrievalEvalSummary) -> dict[str, Any]: """Flatten a RetrievalEvalSummary into a key→value metric dict.""" @@ -65,17 +83,22 @@ def _metrics_from_retrieval(summary: RetrievalEvalSummary) -> dict[str, Any]: def _metrics_from_rag(summary: EvalSummary) -> dict[str, Any]: """Flatten an EvalSummary (RAG) into a key→value metric dict.""" - return { + out: dict[str, Any] = { "total_samples": summary.total_samples, "faithfulness": summary.mean_faithfulness, "relevance": summary.mean_relevance, "pass_rate": summary.pass_rate, "p95_latency_ms": summary.p95_latency_ms, } + if hasattr(summary, "mean_context_precision") and summary.mean_context_precision is not None: + out["context_precision"] = summary.mean_context_precision + if hasattr(summary, "mean_context_recall") and summary.mean_context_recall is not None: + out["context_recall"] = summary.mean_context_recall + return out class EvalRunStore: - """In-memory store for eval runs that supports trend queries and tag filtering. + """In-memory store for eval runs that supports trend queries, tag filtering, and JSON persistence. Parameters ---------- @@ -199,3 +222,63 @@ def compare(self, run_ids: list[str]) -> dict[str, dict[str, Any]]: if missing: raise KeyError(f"Run IDs not found: {missing}") return result + + # -------------------------------------------------------------- persistence + + def to_dict(self) -> dict[str, Any]: + """Serialize the store to a dictionary representation.""" + return { + "max_runs": self._max_runs, + "runs": [r.to_dict() for r in self._runs], + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> EvalRunStore: + """Construct an EvalRunStore from a serialized dictionary.""" + if not isinstance(data, dict): + raise TypeError(f"Expected dict for EvalRunStore, got {type(data).__name__}") + max_runs = int(data.get("max_runs", 1_000)) + store = cls(max_runs=max_runs) + raw_runs = data.get("runs", []) + if isinstance(raw_runs, list): + store._runs = [EvalRun.from_dict(r) for r in raw_runs if isinstance(r, dict)] + return store + + def save_json(self, path: str | Path) -> None: + """Persist recorded runs atomically to a JSON file.""" + file_path = Path(path).resolve() + file_path.parent.mkdir(parents=True, exist_ok=True) + + payload = json.dumps(self.to_dict(), indent=2) + # Write to temporary file in the same directory, then atomic rename + with tempfile.NamedTemporaryFile("w", dir=file_path.parent, delete=False, encoding="utf-8") as tf: + tf.write(payload) + tmp_name = tf.name + + try: + os.replace(tmp_name, file_path) + except Exception: + if os.path.exists(tmp_name): + os.remove(tmp_name) + raise + + @classmethod + def load_json(cls, path: str | Path) -> EvalRunStore: + """Load runs from a JSON file. + + If the file does not exist, returns an empty EvalRunStore. + Supports both `{max_runs: ..., runs: [...]}` and a bare list `[{...}, ...]`. + """ + file_path = Path(path).resolve() + if not file_path.exists(): + return cls() + + with open(file_path, encoding="utf-8") as f: + data = json.load(f) + + if isinstance(data, list): + store = cls() + store._runs = [EvalRun.from_dict(r) for r in data if isinstance(r, dict)] + return store + + return cls.from_dict(data) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6342ec7..9f382e0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,5 @@ +import json + from dynavec import __version__, cli @@ -313,3 +315,88 @@ def import_namespace(self, input_source, namespace="default", batch_size=100): assert result == 1 assert "[FAIL] Import failed: DDB batch write failure" in capsys.readouterr().err + + +# --------------------------------------------------------------------------- +# Eval Check CLI Tests (Issue #305) +# --------------------------------------------------------------------------- + + +def test_eval_check_passes_on_no_regression(tmp_path, capsys): + base_file = tmp_path / "base.json" + curr_file = tmp_path / "curr.json" + base_file.write_text(json.dumps({"metrics": {"recall@10": 0.85, "mrr": 0.70}}), encoding="utf-8") + curr_file.write_text(json.dumps({"metrics": {"recall@10": 0.86, "mrr": 0.70}}), encoding="utf-8") + + exit_code = cli.main(["eval", "check", "--current", str(curr_file), "--baseline", str(base_file)]) + assert exit_code == 0 + out = capsys.readouterr().out + assert "PASSED: No metric regressions detected." in out + + +def test_eval_check_fails_on_regression(tmp_path, capsys): + base_file = tmp_path / "base.json" + curr_file = tmp_path / "curr.json" + base_file.write_text(json.dumps({"metrics": {"recall@10": 0.90}}), encoding="utf-8") + curr_file.write_text(json.dumps({"metrics": {"recall@10": 0.75}}), encoding="utf-8") + + exit_code = cli.main(["eval", "check", "--current", str(curr_file), "--baseline", str(base_file), "--max-drop", "0.05"]) + assert exit_code == 1 + out = capsys.readouterr().out + assert "FAILED: 1 metric(s) regressed beyond threshold." in out + assert "REGRESSION" in out + + +def test_eval_check_json_output(tmp_path, capsys): + base_file = tmp_path / "base.json" + curr_file = tmp_path / "curr.json" + base_file.write_text(json.dumps({"metrics": {"recall@10": 0.90}}), encoding="utf-8") + curr_file.write_text(json.dumps({"metrics": {"recall@10": 0.90}}), encoding="utf-8") + + exit_code = cli.main(["eval", "check", "--current", str(curr_file), "--baseline", str(base_file), "--json"]) + assert exit_code == 0 + out = capsys.readouterr().out + data = json.loads(out) + assert data["passed"] is True + assert "recall@10" in data["diffs"] + + +def test_eval_check_allow_missing_baseline(tmp_path, capsys): + curr_file = tmp_path / "curr.json" + curr_file.write_text(json.dumps({"metrics": {"recall@10": 0.90}}), encoding="utf-8") + missing_baseline = tmp_path / "missing_base.json" + + exit_code = cli.main(["eval", "check", "--current", str(curr_file), "--baseline", str(missing_baseline), "--allow-missing-baseline"]) + assert exit_code == 0 + out = capsys.readouterr().out + assert "[WARN] Baseline file not found" in out + + +def test_eval_check_missing_current_file_returns_code_2(capsys): + exit_code = cli.main(["eval", "check", "--current", "non_existent_file.json", "--baseline", "base.json"]) + assert exit_code == 2 + err = capsys.readouterr().err + assert "[FAIL] Current eval file not found" in err + + +def test_eval_check_records_to_store(tmp_path, capsys): + base_file = tmp_path / "base.json" + curr_file = tmp_path / "curr.json" + store_file = tmp_path / "eval_store.json" + + base_file.write_text(json.dumps({"run_id": "base-1", "metrics": {"recall@10": 0.85}}), encoding="utf-8") + curr_file.write_text(json.dumps({"run_id": "curr-1", "metrics": {"recall@10": 0.88}}), encoding="utf-8") + + exit_code = cli.main([ + "eval", "check", + "--current", str(curr_file), + "--baseline", str(base_file), + "--store", str(store_file), + "--tag", "env=ci", + "--tag", "branch=main" + ]) + assert exit_code == 0 + assert store_file.exists() + store_data = json.loads(store_file.read_text(encoding="utf-8")) + assert len(store_data["runs"]) == 1 + assert store_data["runs"][0]["tags"]["branch"] == "main" diff --git a/tests/test_eval_gate.py b/tests/test_eval_gate.py new file mode 100644 index 0000000..2b8ad70 --- /dev/null +++ b/tests/test_eval_gate.py @@ -0,0 +1,282 @@ +"""Tests for eval regression gating and CI checks.""" + +from __future__ import annotations + +import json + +import pytest + +from dynavec.eval import ( + MetricDirection, + assert_no_regression, + check_regression, +) +from dynavec.eval.gate import get_metric_direction +from dynavec.eval.retrieval import RetrievalEvalSummary +from dynavec.eval.runner import EvalSummary + + +def _retrieval_summary(recall_10: float, ndcg_10: float, mrr: float = 0.8, p95_latency: float = 30.0) -> RetrievalEvalSummary: + return RetrievalEvalSummary( + total_queries=10, + k_values=[5, 10], + mean_recall={5: recall_10 * 0.8, 10: recall_10}, + mean_mrr=mrr, + mean_ndcg={5: ndcg_10 * 0.9, 10: ndcg_10}, + mean_precision={5: 0.5, 10: 0.4}, + p50_latency_ms=12.0, + p95_latency_ms=p95_latency, + ) + + +def _rag_summary(faithfulness: float, relevance: float = 0.85, p95_latency: float = 50.0) -> EvalSummary: + return EvalSummary( + total_samples=20, + mean_faithfulness=faithfulness, + mean_relevance=relevance, + pass_rate=0.9, + pass_threshold=0.7, + p95_latency_ms=p95_latency, + ) + + +# --------------------------------------------------------------------------- +# 1. Basic Pass / Fail +# --------------------------------------------------------------------------- + +def test_identical_runs_pass(): + base = _retrieval_summary(0.85, 0.80) + curr = _retrieval_summary(0.85, 0.80) + report = check_regression(curr, base) + assert report.passed + assert len(report.regressions) == 0 + + +def test_improved_runs_pass(): + base = _retrieval_summary(0.80, 0.75, p95_latency=40.0) + curr = _retrieval_summary(0.90, 0.85, p95_latency=30.0) + report = check_regression(curr, base) + assert report.passed + assert report.diffs["recall@10"].status == "IMPROVED" + assert report.diffs["p95_latency_ms"].status == "IMPROVED" + + +def test_higher_is_better_drop_triggers_regression(): + base = {"recall@10": 0.90} + curr = {"recall@10": 0.80} + report = check_regression(curr, base, max_drop=0.05) + assert not report.passed + assert len(report.regressions) == 1 + assert report.regressions[0].metric == "recall@10" + assert report.regressions[0].is_regression is True + assert report.regressions[0].status == "REGRESSION" + + +def test_higher_is_better_within_threshold_passes(): + base = {"recall@10": 0.90} + curr = {"recall@10": 0.88} + report = check_regression(curr, base, max_drop=0.05) + assert report.passed + assert len(report.regressions) == 0 + + +# --------------------------------------------------------------------------- +# 2. Lower-Is-Better Polarity (Latency, Error Rate) +# --------------------------------------------------------------------------- + +def test_latency_increase_triggers_regression(): + base = {"p95_latency_ms": 100.0} + curr = {"p95_latency_ms": 120.0} + report = check_regression(curr, base, max_drop=0.10) + assert not report.passed + assert len(report.regressions) == 1 + assert report.regressions[0].metric == "p95_latency_ms" + assert report.regressions[0].direction == MetricDirection.LOWER_IS_BETTER + assert report.regressions[0].status == "REGRESSION" + + +def test_latency_reduction_is_improvement(): + base = {"p95_latency_ms": 100.0} + curr = {"p95_latency_ms": 75.0} + report = check_regression(curr, base, max_drop=0.05) + assert report.passed + assert report.diffs["p95_latency_ms"].status == "IMPROVED" + + +def test_compound_latency_detected_as_lower_is_better(): + assert get_metric_direction("p99_latency_ms") == MetricDirection.LOWER_IS_BETTER + assert get_metric_direction("mean_latency_ms") == MetricDirection.LOWER_IS_BETTER + assert get_metric_direction("retrieval_duration_seconds") == MetricDirection.LOWER_IS_BETTER + assert get_metric_direction("system_error_rate") == MetricDirection.LOWER_IS_BETTER + assert get_metric_direction("total_cost") == MetricDirection.LOWER_IS_BETTER + assert get_metric_direction("accuracy") == MetricDirection.HIGHER_IS_BETTER + + +# --------------------------------------------------------------------------- +# 3. Per-Metric Threshold Overrides +# --------------------------------------------------------------------------- + +def test_per_metric_threshold_overrides(): + base = {"recall@10": 0.90, "ndcg@10": 0.80} + curr = {"recall@10": 0.86, "ndcg@10": 0.75} + + report = check_regression( + curr, + base, + max_drop=0.10, + metric_thresholds={"ndcg@10": 0.03}, + ) + assert not report.passed + assert len(report.regressions) == 1 + assert report.regressions[0].metric == "ndcg@10" + assert report.diffs["recall@10"].status == "PASS" + + +# --------------------------------------------------------------------------- +# 4. Absolute vs Relative Drop +# --------------------------------------------------------------------------- + +def test_absolute_drop_mode(): + base = {"faithfulness": 0.50} + curr = {"faithfulness": 0.44} + + report_rel = check_regression(curr, base, max_drop=0.10, relative=True) + assert not report_rel.passed + + report_abs = check_regression(curr, base, max_drop=0.07, relative=False) + assert report_abs.passed + + +# --------------------------------------------------------------------------- +# 5. Edge Cases (Zero Baseline, NaN, Zero Common Metrics, Epsilon) +# --------------------------------------------------------------------------- + +def test_zero_baseline_does_not_crash_division(): + base = {"mrr": 0.0} + curr = {"mrr": 0.0} + report = check_regression(curr, base, max_drop=0.05) + assert report.passed + assert report.diffs["mrr"].relative_change == 0.0 + + +def test_zero_baseline_drop_detection(): + base = {"mrr": 0.0} + curr = {"mrr": 0.1} + report = check_regression(curr, base, max_drop=0.05) + assert report.passed + assert report.diffs["mrr"].status == "IMPROVED" + + +def test_floating_point_epsilon_precision(): + base = {"score": 0.85} + curr = {"score": 0.8075} + report = check_regression(curr, base, max_drop=0.05) + assert report.passed + + +def test_nan_metric_triggers_regression(): + base = {"recall@10": 0.85} + curr = {"recall@10": float("nan")} + report = check_regression(curr, base, max_drop=0.05) + assert not report.passed + assert len(report.regressions) == 1 + assert "NaN" in report.regressions[0].status + + # Baseline is NaN + base_nan = {"recall@10": float("nan")} + curr_valid = {"recall@10": 0.85} + report_base_nan = check_regression(curr_valid, base_nan, max_drop=0.05) + assert not report_base_nan.passed + assert "NaN" in report_base_nan.regressions[0].status + + +def test_zero_common_metrics_fails_regression_by_default(): + base = {"retrieval_recall": 0.80} + curr = {"rag_faithfulness": 0.90} + report = check_regression(curr, base) + assert not report.passed + assert "FAILED" in report.summary_table() + + +def test_zero_common_metrics_passes_when_allow_empty(): + base = {"retrieval_recall": 0.80} + curr = {"rag_faithfulness": 0.90} + report = check_regression(curr, base, allow_empty=True) + assert report.passed + + +def test_fail_on_missing_metric_flag(): + base = {"recall@10": 0.90} + curr = {"faithfulness": 0.85} + + report = check_regression(curr, base, max_drop=0.05, allow_empty=True) + assert report.passed + + report_strict = check_regression( + curr, + base, + metric_thresholds={"recall@10": 0.05}, + fail_on_missing_metric=True, + ) + assert not report_strict.passed + assert len(report_strict.regressions) == 1 + assert "missing" in report_strict.regressions[0].status + + +# --------------------------------------------------------------------------- +# 6. JSON Files & Dict Snapshots +# --------------------------------------------------------------------------- + +def test_check_regression_from_json_files(tmp_path): + base_file = tmp_path / "base.json" + curr_file = tmp_path / "curr.json" + + base_summary = _rag_summary(0.95, 0.90) + curr_summary = _rag_summary(0.70, 0.90) + + base_file.write_text(json.dumps(base_summary.to_dict()), encoding="utf-8") + curr_file.write_text(json.dumps(curr_summary.to_dict()), encoding="utf-8") + + report = check_regression(curr_file, base_file, max_drop=0.05) + assert not report.passed + assert len(report.regressions) == 1 + assert report.regressions[0].metric == "faithfulness" + + +# --------------------------------------------------------------------------- +# 7. Summary Table & Assert Helper +# --------------------------------------------------------------------------- + +def test_summary_table_formatting(): + base = {"recall@10": 0.90, "p95_latency_ms": 50.0} + curr = {"recall@10": 0.80, "p95_latency_ms": 60.0} + report = check_regression(curr, base, max_drop=0.05) + + table = report.summary_table() + assert "Metric" in table + assert "recall@10" in table + assert "p95_latency_ms" in table + assert "REGRESSION" in table + assert "FAILED:" in table + + +def test_summary_table_formats_nan_cleanly(): + base = {"recall@10": 0.90} + curr = {"recall@10": float("nan")} + report = check_regression(curr, base, max_drop=0.05) + table = report.summary_table() + assert "NaN" in table + + +def test_assert_no_regression_raises(): + base = {"recall@10": 0.90} + curr = {"recall@10": 0.70} + + with pytest.raises(AssertionError, match="Eval metric regression detected"): + assert_no_regression(curr, base, max_drop=0.05) + + +def test_assert_no_regression_passes(): + base = {"recall@10": 0.90} + curr = {"recall@10": 0.92} + assert_no_regression(curr, base, max_drop=0.05) diff --git a/tests/test_eval_trend.py b/tests/test_eval_trend.py index 1bea2d7..c4e0190 100644 --- a/tests/test_eval_trend.py +++ b/tests/test_eval_trend.py @@ -1,7 +1,10 @@ -"""Tests for EvalRunStore — trend tracking, tag filtering, and run comparison.""" +"""Tests for EvalRunStore — trend tracking, tag filtering, run comparison, and persistence.""" from __future__ import annotations +import json +from pathlib import Path + import pytest from dynavec.eval import EvalRun, EvalRunStore @@ -111,7 +114,6 @@ def test_filter_no_match_returns_empty(): def test_trend_recall_returns_sorted_time_series(): store = EvalRunStore() - # record three runs at explicit timestamps store.record(_retrieval_summary(0.70, 0.65), ts=1000.0) store.record(_retrieval_summary(0.80, 0.75), ts=2000.0) store.record(_retrieval_summary(0.90, 0.85), ts=3000.0) @@ -119,7 +121,6 @@ def test_trend_recall_returns_sorted_time_series(): series = store.trend("recall", k=10) assert len(series) == 3 - # oldest first assert series[0] == pytest.approx((1000.0, 0.70)) assert series[1] == pytest.approx((2000.0, 0.80)) assert series[2] == pytest.approx((3000.0, 0.90)) @@ -150,12 +151,11 @@ def test_trend_with_tag_filter(): def test_trend_skips_runs_without_metric(): store = EvalRunStore() - # RAG summary has no recall@10 store.record(_rag_summary(0.9), ts=100.0) store.record(_retrieval_summary(0.85, 0.80), ts=200.0) series = store.trend("recall", k=10) - assert len(series) == 1 # only the retrieval run has recall@10 + assert len(series) == 1 assert series[0][1] == pytest.approx(0.85) @@ -183,14 +183,66 @@ def test_compare_missing_id_raises(): # --------------------------------------------------------------------------- -# 5. to_dict round-trip +# 5. Serialization & Persistence # --------------------------------------------------------------------------- -def test_eval_run_to_dict_is_serialisable(): - import json - store = EvalRunStore() - run = store.record(_retrieval_summary(0.82, 0.78), tags={"dataset": "squad"}) +def test_eval_run_to_dict_and_from_dict(): + run = EvalRun( + run_id="run-123", + ts=1700000000.0, + tags={"dataset": "squad", "version": "1.0"}, + metrics={"recall@10": 0.88, "p95_latency_ms": 25.0}, + ) d = run.to_dict() - json.dumps(d) # must not raise - assert d["tags"]["dataset"] == "squad" - assert "recall@10" in d["metrics"] + restored = EvalRun.from_dict(d) + + assert restored.run_id == run.run_id + assert restored.ts == run.ts + assert restored.tags == run.tags + assert restored.metrics == run.metrics + + +def test_eval_run_store_save_and_load_json(tmp_path: Path): + store = EvalRunStore(max_runs=50) + store.record(_retrieval_summary(0.82, 0.78), tags={"dataset": "squad", "version": "v1"}) + store.record(_rag_summary(0.94), tags={"model": "claude-3-opus", "version": "v2"}) + + file_path = tmp_path / "subdir" / "eval_store.json" + store.save_json(file_path) + assert file_path.exists() + + loaded = EvalRunStore.load_json(file_path) + assert len(loaded.all_runs()) == 2 + assert loaded._max_runs == 50 + + v1_runs = loaded.filter(version="v1") + assert len(v1_runs) == 1 + assert v1_runs[0].metrics["recall@10"] == pytest.approx(0.82) + + v2_runs = loaded.filter(version="v2") + assert len(v2_runs) == 1 + assert v2_runs[0].metrics["faithfulness"] == pytest.approx(0.94) + + +def test_eval_run_store_load_nonexistent_returns_empty(tmp_path: Path): + missing_path = tmp_path / "does_not_exist.json" + loaded = EvalRunStore.load_json(missing_path) + assert isinstance(loaded, EvalRunStore) + assert len(loaded.all_runs()) == 0 + + +def test_eval_run_store_load_bare_list(tmp_path: Path): + file_path = tmp_path / "bare_list.json" + raw_list = [ + { + "run_id": "r1", + "ts": 1000.0, + "tags": {"env": "prod"}, + "metrics": {"recall@10": 0.85}, + } + ] + file_path.write_text(json.dumps(raw_list), encoding="utf-8") + + loaded = EvalRunStore.load_json(file_path) + assert len(loaded.all_runs()) == 1 + assert loaded.all_runs()[0].run_id == "r1"