feat(eval): add regression runs and CI gating (#305)

docs/eval_ci_gating.md

@@ -0,0 +1,165 @@
+# Evaluation Regression Runs & CI Gating
+
+`dynaevals` includes built-in regression detection and CI gating to prevent performance drops (retrieval quality, faithfulness, relevance, or latency regressions) from slipping into production.
+
+---
+
+## 1. Overview
+
+When developing agent workflows, RAG pipelines, or retrieval systems, code changes, chunking adjustments, or prompt changes can silently degrade search accuracy or answer quality.
+
+Regression gating solves this by comparing an **evaluation run** against an established **baseline**:
+- **Automatic Polarity**: Quality metrics (`recall@k`, `mrr`, `ndcg@k`, `faithfulness`, `relevance`) must not decrease below a threshold. Latency metrics (`p50_latency_ms`, `p95_latency_ms`) must not increase beyond a threshold.
+- **CI Gate**: Non-zero exit code (`1`) immediately fails CI builds when any gated metric regresses.
+- **Trend History**: Persists evaluation snapshots across versions/commits in `EvalRunStore`.
+
+---
+
+## 2. CLI Usage (`dynavec eval check`)
+
+The `dynavec eval check` command compares a current eval result against a baseline JSON snapshot.
+
+```bash
+dynavec eval check \
+  --current current_eval.json \
+  --baseline baseline_eval.json \
+  --max-drop 0.05
+```
+
+### CLI Arguments
+
+| Flag | Description | Default |
+|------|-------------|---------|
+| `--current`, `-c` | Path to current run eval summary JSON | **Required** |
+| `--baseline`, `-b` | Path to baseline run eval summary JSON | **Required** (unless `--allow-missing-baseline`) |
+| `--max-drop` | Max allowed relative degradation | `0.05` (5%) |
+| `--threshold METRIC=VAL` | Override threshold for a specific metric (repeatable) | None |
+| `--absolute` | Compare absolute drop instead of relative change | `False` |
+| `--store FILE` | Path to `EvalRunStore` JSON file to persist history | None |
+| `--tag KEY=VALUE` | Attach metadata tag to the stored run (repeatable) | None |
+| `--json` | Emit structured JSON report instead of text table | `False` |
+| `--allow-missing-baseline` | Succeed with warning if baseline file does not exist | `False` |
+
+### Exit Codes
+
+- `0`: All metrics passed (within thresholds or improved).
+- `1`: One or more metrics regressed beyond threshold (CI fails).
+- `2`: Missing file or invalid CLI arguments.
+
+### Output Example
+
+```text
+Metric         | Baseline | Current | Delta   | % Change | Threshold | Status
+---------------+----------+---------+---------+----------+-----------+-----------
+mrr            | 0.8200   | 0.8350  | +0.0150 | +1.83%   | <= 5.0%   | IMPROVED
+p95_latency_ms | 32.0000  | 33.5000 | +1.5000 | +4.69%   | <= 5.0%   | PASS
+recall@10      | 0.9100   | 0.8200  | -0.0900 | -9.89%   | <= 5.0%   | REGRESSION
+
+Overall: FAILED: 1 metric(s) regressed beyond threshold.
+```
+
+---
+
+## 3. GitHub Actions Workflow Example
+
+Create `.github/workflows/eval-gate.yml` in your repository:
+
+```yaml
+name: Evaluation Regression Gate
+
+on:
+  pull_request:
+    branches: [main, development]
+  push:
+    branches: [main]
+
+jobs:
+  eval-gate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Cache eval baseline
+        id: cache-baseline
+        uses: actions/cache@v4
+        with:
+          path: .eval_cache/baseline.json
+          key: eval-baseline-${{ runner.os }}
+
+      - name: Run benchmark evaluation
+        run: |
+          python -m tests.run_eval_benchmark --output current_eval.json
+
+      - name: Gate on regressions
+        run: |
+          dynavec eval check \
+            --current current_eval.json \
+            --baseline .eval_cache/baseline.json \
+            --max-drop 0.05 \
+            --threshold p95_latency_ms=0.15 \
+            --allow-missing-baseline \
+            --store .eval_cache/eval_history.json \
+            --tag commit=${{ github.sha }} \
+            --tag branch=${{ github.ref_name }}
+
+      - name: Update baseline on main branch
+        if: github.ref == 'refs/heads/main' && success()
+        run: |
+          mkdir -p .eval_cache
+          cp current_eval.json .eval_cache/baseline.json
+```
+
+---
+
+## 4. Python Testing API (`assert_no_regression`)
+
+You can also enforce regression gates inside `pytest` test suites:
+
+```python
+from dynavec.eval import assert_no_regression, EvalRunner, MockJudge
+
+def test_rag_quality_no_regression():
+    runner = EvalRunner(judge=MockJudge())
+    current_summary = runner.run(test_dataset)
+
+    # Compare against saved baseline
+    assert_no_regression(
+        current=current_summary,
+        baseline="benchmarks/baselines/v0.7.0.json",
+        max_drop=0.05,
+        metric_thresholds={"faithfulness": 0.03},
+    )
+```
+
+---
+
+## 5. Cross-Version Trend Persistence (`EvalRunStore`)
+
+`EvalRunStore` can persist evaluation history to disk and reload it:
+
+```python
+from dynavec.eval import EvalRunStore
+
+# Load existing history (or create new if file doesn't exist)
+store = EvalRunStore.load_json("eval_history.json")
+
+# Record new evaluation run
+store.record(summary, tags={"version": "0.7.1", "git_commit": "abc1234"})
+
+# Persist to disk atomically
+store.save_json("eval_history.json")
+
+# Query trends over time
+recall_trend = store.trend("recall", k=10)  # [(timestamp, value), ...]
+```

src/dynavec/cli.py

@@ -86,6 +86,51 @@ def _parser() -> argparse.ArgumentParser:
 		default=8000,
 		help="Port for SSE transport (default: 8000)",
 	)
+
+	eval_cmd = subparsers.add_parser("eval", help="evaluation regression gating and trend commands")
+	eval_sub = eval_cmd.add_subparsers(dest="eval_command")
+	eval_check = eval_sub.add_parser(
+		"check", help="compare eval run against baseline and fail on regression"
+	)
+	eval_check.add_argument("--current", "-c", required=True, help="path to current eval run JSON file")
+	eval_check.add_argument(
+		"--baseline", "-b", required=False, help="path to baseline eval run JSON file"
+	)
+	eval_check.add_argument(
+		"--max-drop",
+		type=float,
+		default=0.05,
+		help="max allowed relative drop (default: 0.05)",
+	)
+	eval_check.add_argument(
+		"--threshold",
+		action="append",
+		dest="thresholds",
+		metavar="METRIC=VAL",
+		help="per-metric threshold override, e.g. --threshold recall@10=0.02 (repeatable)",
+	)
+	eval_check.add_argument(
+		"--absolute",
+		action="store_true",
+		help="use absolute drop threshold instead of relative",
+	)
+	eval_check.add_argument(
+		"--store",
+		help="path to EvalRunStore JSON file to record/persist runs",
+	)
+	eval_check.add_argument(
+		"--tag",
+		action="append",
+		dest="tags",
+		metavar="KEY=VALUE",
+		help="metadata tags to attach to the recorded run (repeatable)",
+	)
+	eval_check.add_argument("--json", action="store_true", help="output report as JSON")
+	eval_check.add_argument(
+		"--allow-missing-baseline",
+		action="store_true",
+		help="succeed with warning if baseline file does not exist",
+	)
 	return parser
 
 
@@ -280,6 +325,99 @@ def _check_dynamodb(session: Any, table: str, region: str | None) -> str:
 	return "Accessible"
 
 
+def _eval_check(args: argparse.Namespace) -> int:
+	import json
+	from pathlib import Path
+
+	from .eval.gate import check_regression
+	from .eval.trend import EvalRun, EvalRunStore
+
+	current_path = Path(args.current)
+	if not current_path.exists():
+		print(f"[FAIL] Current eval file not found: {args.current}", file=sys.stderr)
+		return 2
+
+	tag_dict: dict[str, str] = {}
+	if args.tags:
+		for item in args.tags:
+			if "=" in item:
+				k, v = item.split("=", 1)
+				tag_dict[k.strip()] = v.strip()
+
+	if args.store:
+		try:
+			import time
+			import uuid
+
+			from .eval.gate import _extract_metrics
+
+			store = EvalRunStore.load_json(args.store)
+			curr_metrics, c_run_id = _extract_metrics(current_path)
+			run_id = c_run_id or uuid.uuid4().hex[:12]
+			run = EvalRun(
+				run_id=run_id,
+				ts=time.time(),
+				tags=tag_dict,
+				metrics=curr_metrics,
+			)
+			store._runs.append(run)
+			store.save_json(args.store)
+		except Exception as exc:
+			print(f"[WARN] Failed to update trend store at {args.store}: {exc}", file=sys.stderr)
+
+	if not args.baseline:
+		if args.allow_missing_baseline:
+			print(
+				"[WARN] No baseline provided and --allow-missing-baseline set. Skipping regression check."
+			)
+			return 0
+		print(
+			"[FAIL] Missing required --baseline argument (or pass --allow-missing-baseline).",
+			file=sys.stderr,
+		)
+		return 2
+
+	baseline_path = Path(args.baseline)
+	if not baseline_path.exists():
+		if args.allow_missing_baseline:
+			print(
+				f"[WARN] Baseline file not found: {args.baseline}. Bootstrapping mode: skipping regression check."
+			)
+			return 0
+		print(f"[FAIL] Baseline file not found: {args.baseline}", file=sys.stderr)
+		return 2
+
+	metric_thresholds: dict[str, float] = {}
+	if args.thresholds:
+		for item in args.thresholds:
+			if "=" in item:
+				m, val = item.split("=", 1)
+				try:
+					metric_thresholds[m.strip()] = float(val.strip())
+				except ValueError:
+					print(f"[FAIL] Invalid threshold value for {m}: {val}", file=sys.stderr)
+					return 2
+
+	try:
+		report = check_regression(
+			current=current_path,
+			baseline=baseline_path,
+			max_drop=args.max_drop,
+			metric_thresholds=metric_thresholds,
+			relative=not args.absolute,
+		)
+	except Exception as exc:
+		print(f"[FAIL] Regression check error: {exc}", file=sys.stderr)
+		return 2
+
+	if args.json:
+		print(json.dumps(report.to_dict(), indent=2))
+	else:
+		print(report.summary_table())
+
+	return 0 if report.passed else 1
+
+
 def main(argv: list[str] | None = None) -> int:
 	args = _parser().parse_args(argv)
 	if args.command == "version":
@@ -301,6 +439,11 @@ def main(argv: list[str] | None = None) -> int:
 		else:
 			server.run(transport="stdio")
 		return 0
+	if args.command == "eval":
+		if getattr(args, "eval_command", None) == "check":
+			return _eval_check(args)
+		_parser().parse_args(["eval", "--help"])
+		return 0
 	_parser().print_help()
 	return 0
 

src/dynavec/eval/__init__.py

@@ -1,4 +1,4 @@
-"""Evaluation framework for RAG generation quality, context quality, and retrieval metrics."""
+"""Evaluation framework for RAG generation quality, context quality, retrieval metrics, and CI regression gating."""
 
 from __future__ import annotations
 
@@ -14,6 +14,13 @@
     extract_json,
 )
 from .chart import plot_eval_summary, plot_retrieval_metrics
+from .gate import (
+    MetricDiff,
+    MetricDirection,
+    RegressionReport,
+    assert_no_regression,
+    check_regression,
+)
 from .judges import (
     BedrockJudge,
     ChatModelJudge,
@@ -48,6 +55,12 @@
 __all__ = [
     "EvalRun",
     "EvalRunStore",
+    # CI Regression Gating
+    "MetricDirection",
+    "MetricDiff",
+    "RegressionReport",
+    "check_regression",
+    "assert_no_regression",
     # LLM Judge & Data Models
     "BaseJudge",
     "OpenAIJudge",