From 2b70eeb545c4cd68721df284612765285b707f15 Mon Sep 17 00:00:00 2001 From: vchamarthi <123653452+vchamarthi@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:26:55 -0600 Subject: [PATCH 1/6] feat(report): one summary over cli_output.v1 documents, the leaves it needs, and actions/report (#749) - report summary: merges cli_output.v1 documents into markdown, json (report_summary.v1) or annotations. Envelope keys only; missing, unreadable or foreign inputs stay rows (not_run / invalid). No verdict computed, exit 0 once written. Stdlib only, no GITHUB_* read. - catalog check, catalog eval and skill routing take --json. A subject may carry findings (level, code, message, file, line); catalog check, catalog eval, repo validate and repo scan fill them. - actions/report: renders out-dir into the job summary and annotations, optional sticky PR comment, uploads out-dir. repo-eval.yml renders repo validate and repo scan with the same renderer. Signed-off-by: vchamarthi <123653452+vchamarthi@users.noreply.github.com> --- .github/workflows/repo-eval.yml | 54 +-- actions/report/action.yml | 74 ++++ ...261116_120000_vchamarthi_report-summary.md | 6 + dokimi/docs/reference/cli-interface.json | 15 + dokimi/dokimi/cli_output.py | 14 + dokimi/dokimi/cli_routes.py | 1 + dokimi/dokimi/commands/catalog.py | 55 ++- dokimi/dokimi/commands/catalog_eval.py | 44 ++- dokimi/dokimi/commands/novelty.py | 44 ++- dokimi/dokimi/commands/repo.py | 10 +- dokimi/dokimi/commands/repo_scan.py | 17 + dokimi/dokimi/commands/report.py | 26 +- dokimi/dokimi/commands/routing_parser.py | 8 +- dokimi/dokimi/report/summary.py | 177 ++++++++++ dokimi/dokimi/schemas/cli_output.v1.json | 317 +++++++++++++++--- dokimi/tests/test_report_summary.py | 210 ++++++++++++ 16 files changed, 976 insertions(+), 96 deletions(-) create mode 100644 actions/report/action.yml create mode 100644 dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md create mode 100644 dokimi/dokimi/report/summary.py create mode 100644 dokimi/tests/test_report_summary.py diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index a297dd0b1..91361576b 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -166,22 +166,9 @@ jobs: status=0 dokimi repo validate . --manifest "$MANIFEST" || status=$? dokimi repo validate . --manifest "$MANIFEST" --json > "$RUNNER_TEMP/validate.json" || true - python - <<'PY' >> "$GITHUB_STEP_SUMMARY" - import json, os - path = os.path.join(os.environ["RUNNER_TEMP"], "validate.json") - try: - d = json.load(open(path, encoding="utf-8"))["data"] # cli_output.v1: the report under data - d["tasks"] # a run that failed has data null - except (OSError, ValueError, KeyError, TypeError): - print("### repo validate", "", "The manifest could not be read; see the job log.", sep="\n") - raise SystemExit - tasks = d["tasks"] - print("### repo validate", "", f"{'ok' if d['ok'] else 'FAILED'}: `{d['manifest']['manifest']}`, " - f"{tasks['count']} task(s), {len(d['guidance_files'])} guidance file(s)", sep="\n") - for kind in ("errors", "warnings", "notes"): - for line in d[kind]: - print(f"- {kind[:-1]}: {line}") - PY + # One renderer over the cli_output.v1 document (#749): the job summary and an annotation per finding. + dokimi report summary "$RUNNER_TEMP/validate.json" >> "$GITHUB_STEP_SUMMARY" + dokimi report summary "$RUNNER_TEMP/validate.json" --format annotations exit "$status" - name: Select tasks id: select @@ -335,36 +322,21 @@ jobs: env: MANIFEST: ${{ inputs.manifest }} run: | - dokimi repo scan . --manifest "$MANIFEST" --out "$RUNNER_TEMP/repo-readiness.json" - python - <<'PY' >> "$GITHUB_STEP_SUMMARY" - import json, os - d = json.load(open(os.path.join(os.environ["RUNNER_TEMP"], "repo-readiness.json"), encoding="utf-8")) - c = d["counts"] - - def cell(text): - return str(text).replace("|", "\\|").replace("\n", " ") - - print("### repo scan (L0, report only)", "", - f"{len(d['files'])} guidance file(s), adapters: {', '.join(d['adapters']) or 'none'}", "", - "| claims | ok | unresolved | fail |", "|---|---|---|---|", - f"| {c['total']} | {c['ok']} | {c['unresolved']} | {c['fail']} |", sep="\n") - for w in d["warnings"]: - print(f"- warning: {cell(w)}") - fails = [x for x in d["claims"] if x["status"].split("_", 1)[0] == "fail"] - if fails: - print("", "| file | line | kind | claim | why |", "|---|---|---|---|---|", sep="\n") - for x in fails[:100]: - print(f"| {cell(x['file'])} | {x['line']} | {cell(x.get('kind'))} | " - f"`{cell(str(x.get('claim'))[:80])}` | {cell(x.get('why'))} |") - if len(fails) > 100: - print(f"\n{len(fails) - 100} more in the repo-readiness.json artifact") - PY + status=0 + dokimi repo scan . --manifest "$MANIFEST" --out "$RUNNER_TEMP/repo-readiness.json" \ + --json > "$RUNNER_TEMP/repo-scan.json" || status=$? + # The claim counts, and each failing claim at its file and line, from the one renderer (#749). + dokimi report summary "$RUNNER_TEMP/repo-scan.json" >> "$GITHUB_STEP_SUMMARY" + dokimi report summary "$RUNNER_TEMP/repo-scan.json" --format annotations + exit "$status" - name: Upload the scan report if: ${{ !cancelled() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: repo-eval-scan-${{ needs.validate.outputs.id }} - path: ${{ runner.temp }}/repo-readiness.json + path: | + ${{ runner.temp }}/repo-readiness.json + ${{ runner.temp }}/repo-scan.json if-no-files-found: ignore tasks-check: diff --git a/actions/report/action.yml b/actions/report/action.yml new file mode 100644 index 000000000..e72b71959 --- /dev/null +++ b/actions/report/action.yml @@ -0,0 +1,74 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: dokimi report +description: >- + Places the summary of every cli_output.v1 document in out-dir: runs + `dokimi report summary`, appends its Markdown to the job summary, prints its + annotations, optionally keeps one sticky PR comment, and uploads out-dir. No + parsing and no verdict in YAML: the gate stays the exit code of each dokimi + step. Needs `contents: read`; `pull-requests: write` only with comment: sticky. + Run it after actions/setup, with `if: always()`. + +inputs: + out-dir: + description: >- + Directory holding the cli_output.v1 documents (`*.json`, not recursive), usually + the out-dir output of actions/setup. The rendered summary goes to its `summary/` + subdirectory. + required: true + comment: + description: "false, or sticky: create or update one comment on the pull request" + required: false + default: 'false' + github-token: + description: Token for the sticky comment (needs pull-requests write) + required: false + default: ${{ github.token }} + artifact-name: + description: Name of the uploaded out-dir artifact; empty skips the upload + required: false + default: dokimi-report + +runs: + using: composite + steps: + - name: Render the summary + shell: bash + env: + OUT: ${{ inputs.out-dir }} + run: | + if ! command -v dokimi >/dev/null; then + echo "::error title=dokimi report::dokimi is not on PATH; run intel/dokimi/actions/setup first" + exit 1 + fi + mkdir -p -- "$OUT/summary" + dokimi report summary --format markdown --out "$OUT/summary/summary.md" -- "$OUT" + dokimi report summary --format json --out "$OUT/summary/summary.json" -- "$OUT" + dokimi report summary --format annotations -- "$OUT" + cat -- "$OUT/summary/summary.md" >> "$GITHUB_STEP_SUMMARY" + - name: Sticky pull request comment + if: ${{ inputs.comment == 'sticky' && github.event.pull_request.number }} + shell: bash + env: + OUT: ${{ inputs.out-dir }} + GH_TOKEN: ${{ inputs.github-token }} + PR: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + marker='' + { echo "$marker"; cat -- "$OUT/summary/summary.md"; } > "$OUT/summary/comment.md" + id=$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | head -n1) + if [ -n "$id" ]; then + gh api -X PATCH "repos/$REPO/issues/comments/$id" -F "body=@$OUT/summary/comment.md" >/dev/null + else + gh api -X POST "repos/$REPO/issues/$PR/comments" -F "body=@$OUT/summary/comment.md" >/dev/null + fi + - name: Upload out-dir + if: ${{ inputs.artifact-name != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.out-dir }} + if-no-files-found: ignore diff --git a/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md b/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md new file mode 100644 index 000000000..360b7c3bc --- /dev/null +++ b/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md @@ -0,0 +1,6 @@ +### Added + +- **`dokimi report summary` merges `cli_output.v1` documents into one summary** (#749, the renderer). It takes files or directories of `*.json` and writes `--format markdown` (a check table and a subject table, usable as a job summary, a PR comment or terminal output), `json` (`report_summary.v1`) or `annotations` (`::error`/`::warning` lines). It reads envelope keys only, so a new leaf needs no change in it. A missing, unreadable or non-`cli_output.v1` input stays a row (`not_run` or `invalid`) and makes the overall state bad; it is never dropped or shown as a pass. It computes no verdict and exits 0 once written: the gate stays each leaf's own exit code. Stdlib only, and it reads no `GITHUB_*` variable. +- **`catalog check`, `catalog eval` and `skill routing` take `--json`** (#749, #94 F2): one `cli_output.v1` document on stdout, with one subject per skill (and per task for `catalog check`). Exit codes are unchanged. `skill routing --json` is report only (`verdict: null`) and carries `queries_self_derived` next to its scores; its human summary goes to stderr. `catalog check --json` refuses `--format json`. +- **A `cli_output.v1` subject may carry `findings`** (`level`, `code`, `message`, `file`, `line`). `catalog check`, `catalog eval`, `repo validate` and `repo scan` fill them; `repo scan --json` puts each failing claim at its file and line and adds the claim counts to `summary`. `report summary --format annotations` gives each finding a `file=`/`line=` annotation, and the Markdown shows the scalar `summary` values of each document. +- **`actions/report`** places the summary: it runs `dokimi report summary` over `out-dir`, appends the Markdown to the job summary, prints the annotations, keeps one sticky PR comment with `comment: sticky`, and uploads `out-dir`. No parsing and no verdict in YAML. `repo-eval.yml` renders `repo validate` and `repo scan` with the same renderer instead of two inline Python blocks. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index fe220c6ea..b04ab7833 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -290,6 +290,7 @@ {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--ignore", "action": "_AppendAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--jobs", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, + {"key": "--json", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--jsonl", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--kind", "action": "_StoreAction", "choices": ["'all'", "'skills'", "'tasks'"], "const": null, "default": "all", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--online", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, @@ -310,6 +311,7 @@ {"key": "catalog_root", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "argument", "nargs": null, "position": 0, "required": true, "type": "pathlib._local.Path#661d9498960e"}, {"key": "--changed-from", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--json", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--out-dir", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": true, "type": "pathlib._local.Path#661d9498960e"}, {"key": "--tier", "action": "_StoreAction", "choices": ["'static'"], "const": null, "default": "static", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} ], @@ -1530,6 +1532,18 @@ "path": "dokimi report model-compare", "subcommand_required": null, "subcommands": {} + }, + "summary": { + "allow_abbrev": true, + "params": [ + {"key": "inputs", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "argument", "nargs": "+", "position": 0, "required": true, "type": null}, + {"key": "--format", "action": "_StoreAction", "choices": ["'annotations'", "'json'", "'markdown'"], "const": null, "default": "markdown", "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--out", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} + ], + "path": "dokimi report summary", + "subcommand_required": null, + "subcommands": {} } } }, @@ -1752,6 +1766,7 @@ {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--include-disabled", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--include-invalid", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, + {"key": "--json", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--keep-copies-under", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--max-tokens", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f"}, {"key": "--max-usd", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": "dokimi.commands.novelty_run.usd_amount#af10a4ceaa08"}, diff --git a/dokimi/dokimi/cli_output.py b/dokimi/dokimi/cli_output.py index 77c7c0a1f..5e42aec84 100644 --- a/dokimi/dokimi/cli_output.py +++ b/dokimi/dokimi/cli_output.py @@ -107,6 +107,20 @@ def emit_leaf(kind: str, data: Any, **fields: Any) -> None: emit(envelope(kind, run_id=current_run_id(), data=data, **fields)) +def finding(level: str, code: str, message: str, file: str | None = None, line: int | None = None) -> dict[str, Any]: + """One entry of a subject's ``findings``. ``warning`` is spelled ``warn``.""" + return {"level": "warn" if level == "warning" else level, "code": code, "message": message, + "file": file, "line": line} + + +def quiet_stdout(): + """While a --json leaf runs, its human-readable prints go to stderr: stdout holds only the document.""" + import contextlib + import sys + + return contextlib.redirect_stdout(sys.stderr) + + def schema_document() -> dict[str, Any]: """The schema itself, for ``dokimi data schema show cli_output.v1``.""" return json.loads((SCHEMA_DIR / f"{SCHEMA_VERSION}.json").read_text(encoding="utf-8")) diff --git a/dokimi/dokimi/cli_routes.py b/dokimi/dokimi/cli_routes.py index d44d6ac72..515b7fc55 100644 --- a/dokimi/dokimi/cli_routes.py +++ b/dokimi/dokimi/cli_routes.py @@ -97,6 +97,7 @@ class Tombstone: Leaf(("report", "eval"), FREE), Leaf(("report", "generate"), FREE), Leaf(("report", "model-compare"), FREE), + Leaf(("report", "summary"), FREE), Leaf(("answers", "generate"), PAID), Leaf(("benchmark", "run"), PAID), Leaf(("benchmark", "preflight"), FREE), diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index 30df4f882..c6def2495 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -167,6 +167,11 @@ def _scan_tasks(args: argparse.Namespace, root: Path, report: dict, as_json: boo return failed +def _refuse_json_with_format_json(args: argparse.Namespace) -> None: + if getattr(args, "json", False) and args.format == "json": + raise InputError("--json prints cli_output.v1 with the report under data; drop --format json") + + def _check_scan_flags(args: argparse.Namespace) -> None: """Raise InputError on a flag combination the scan cannot honour; the per-skill flags need --per-skill.""" from dokimi.external.validate import FINDING_CODES @@ -184,6 +189,7 @@ def _check_scan_flags(args: argparse.Namespace) -> None: if "unreadable" in (args.ignore or ()): # The whole-catalog exit reads the unfiltered findings, --per-skill the filtered rows: ignoring it splits them. raise InputError("--ignore unreadable: a SKILL.md the OS will not read always fails the scan, as a load error does") + _refuse_json_with_format_json(args) if args.jsonl and args.format == "json": raise InputError("--jsonl writes one row per skill to a file; drop --format json") if args.resume and not args.jsonl: @@ -205,7 +211,7 @@ def cmd_catalog_scan(args: argparse.Namespace) -> None: raise InputError(f"--root is not a directory: {root}") _check_scan_flags(args) - as_json = args.format == "json" + as_json = args.format == "json" or getattr(args, "json", False) report: dict = {"root": str(root), "skills": [], "tasks": []} any_failure = False declared = 0 @@ -222,12 +228,50 @@ def cmd_catalog_scan(args: argparse.Namespace) -> None: print(f"no SKILL.md found under {root}", file=sys.stderr) if args.kind == "skills" or args.strict or not report["tasks"]: any_failure = True - if as_json: + _finish_scan(args, report, as_json, any_failure) + + +def _finish_scan(args: argparse.Namespace, report: dict, as_json: bool, failed: bool) -> None: + """Print the JSON report or the cli_output.v1 document, then exit 1 when the scan failed.""" + if getattr(args, "json", False): + _emit_check(args, report, failed) + elif as_json: print(json.dumps(report, indent=2)) - if any_failure: + if failed: raise SystemExit(1) +def _emit_check(args: argparse.Namespace, report: dict, failed: bool) -> None: + """The cli_output.v1 document of `catalog check --json`: one subject per skill and per task, with its findings. + + A subject's verdict is ``fail`` when it alone fails the exit: a skill that does not load, or with + --strict one in ``error`` or granting unrestricted Bash with a risky command; else ``pass``. + """ + from dokimi import cli_output + + subjects = [] + for row in report["skills"]: + file = f"{row['relpath']}/SKILL.md" if row.get("relpath") else row.get("path") + bad = bool(row["error"]) or any(f["code"] == "unreadable" for f in row["findings"]) or ( + args.strict and (row["state"] == "error" or unrestricted_and_risky(row))) + findings = [cli_output.finding(f["level"], f["code"], f["message"], file) for f in row["findings"]] + if row["error"]: + findings.insert(0, cli_output.finding("error", "load-error", str(row["error"]), file)) + subjects.append({"subject": row["name"], "verdict": "fail" if bad else "pass", "findings": findings}) + for row in report["tasks"]: + findings = [cli_output.finding("error", "task-load-error", str(row["reason"]), row["path"])] \ + if row["state"] == "error" else [] + subjects.append({"subject": f"task:{row['name']}", "verdict": "fail" if row["state"] == "error" else "pass", + "findings": findings}) + counts = {state: sum(r["state"] == state for r in report["skills"]) for state in ("ok", "warn", "error")} + reason = None + if failed: + n = sum(s["verdict"] == "fail" for s in subjects) + reason = f"{n} subject(s) fail" + (" under --strict" if args.strict else "") if n else "nothing to scan" + cli_output.emit_leaf("catalog check", report, verdict="fail" if failed else "pass", reason=reason, + summary={"skills": counts, "tasks": len(report["tasks"])}, subjects=subjects) + + def _write_trigger_set(args: argparse.Namespace, queries, catalog_root: Path) -> None: """Write the query set to ``--out`` in ``--format``, or exit 2 naming ``--out`` when it cannot be written.""" import json @@ -535,7 +579,10 @@ def register(sub, positive_int, eval_sub) -> None: "probe got an answer, are warnings; targets not checked are counted in one summary line " "(off by default: no request is made without it; not with --per-skill)", ) - scan_p.set_defaults(func=cmd_catalog_scan) + scan_p.add_argument("--json", action="store_true", + help="Print cli_output.v1 on stdout: the report under data, one subject per skill and task " + "with its findings; the exit code is unchanged") + scan_p.set_defaults(func=cmd_catalog_scan, emits_cli_output=True, cli_kind="catalog check") fm_p = catalog_sub.add_parser( "formats", diff --git a/dokimi/dokimi/commands/catalog_eval.py b/dokimi/dokimi/commands/catalog_eval.py index 387efd8ad..be7a4883a 100644 --- a/dokimi/dokimi/commands/catalog_eval.py +++ b/dokimi/dokimi/commands/catalog_eval.py @@ -190,15 +190,55 @@ def cmd_catalog_eval(args: argparse.Namespace) -> None: if input_error: raise InputError(f"catalog eval: {exc}") from exc raise - print(f"Static catalog checks: {report['status']}; receipts: {out}") + _finish(args, report, out) + + +def _finish(args: argparse.Namespace, report: dict[str, Any], out: Path) -> None: + """The one line, or with --json the document; exit 1 unless the checks pass.""" + if getattr(args, "json", False): + _emit(report, out) + else: + print(f"Static catalog checks: {report['status']}; receipts: {out}") if report["status"] != "pass": raise SystemExit(1) +def _emit(report: dict[str, Any], out: Path) -> None: + """The cli_output.v1 document of `catalog eval --json`: one subject per skill with its static findings. + + ``data`` holds the per-skill evaluation state (``no_tasks``, ``not_measured``, ``checks_only``) and the + findings that belong to no skill; the full report stays in the receipts under ``--out-dir``. + """ + from dokimi import cli_output + from dokimi.external.catalog_eval import shared_problems + + def finding(item: dict[str, Any]) -> dict[str, Any]: + return cli_output.finding(item["level"], item["code"], item["message"], item.get("path")) + + subjects = [{"subject": row["name"], "verdict": row["checks"]["status"], + "findings": [finding(item) for item in row["checks"]["findings"]]} for row in report["skills"]] + shared = shared_problems(report["problems"], report["skills"]) + evaluation = {row["name"]: row["evaluation"] for row in report["skills"]} + states: dict[str, int] = {} + for state in evaluation.values(): + states[state] = states.get(state, 0) + 1 + failed = report["status"] != "pass" + reason = None + if failed: + reason = f"{sum(s['verdict'] == 'fail' for s in subjects)} skill(s) fail, {len(shared)} shared finding(s)" + cli_output.emit_leaf( + "catalog eval", {"evaluation": evaluation, "shared_findings": shared, "plan_digest": report.get("plan_digest")}, + verdict=report["status"], reason=reason, subjects=subjects, + summary={"tier": report["tier"], "skills": len(subjects), "evaluation": states}, + artifacts={"out_dir": str(out), "files": [str(out / name) for name in _OUTPUTS]}) + + def register(sub: Any) -> None: parser = sub.add_parser("eval", help="Plan and check a catalog without running models, Docker or skill scripts") parser.add_argument("catalog_root", type=Path, help="Local catalog checkout") parser.add_argument("--tier", choices=["static"], default="static", help="static: deterministic checks only") parser.add_argument("--out-dir", type=Path, required=True, help="Dedicated directory for plan, result and report") parser.add_argument("--changed-from", help="Local base commit; an unavailable base conservatively checks every skill") - parser.set_defaults(func=cmd_catalog_eval) + parser.add_argument("--json", action="store_true", + help="Print cli_output.v1 on stdout: one subject per skill with its findings; receipts are written as before") + parser.set_defaults(func=cmd_catalog_eval, emits_cli_output=True, cli_kind="catalog eval") diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 11bba7747..33d2c19c5 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -379,7 +379,7 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: print_all_vs_all_summary(*summary) -def cmd_routing(args: argparse.Namespace) -> None: +def cmd_routing(args: argparse.Namespace) -> tuple[dict, Path] | None: """Can a skill (or each skill under a root) be selected against its competitors? Lexical floor, model on top.""" from dokimi.novelty import cost if args.repeats > 1 and not args.model: @@ -391,7 +391,8 @@ def cmd_routing(args: argparse.Namespace) -> None: ) if args.all_vs_all: - return _cmd_routing_all_vs_all(args) + _cmd_routing_all_vs_all(args) + return None gold, body, competitors, left_out, conflicts = load_routing_field(args) # Checked before the query file is read, so the message is about the real problem. if not competitors and conflicts: @@ -403,7 +404,7 @@ def cmd_routing(args: argparse.Namespace) -> None: if args.estimate_only: refuse_estimate_without_queries(args, queries, self_derived) print_routing_estimate(args, candidates, queries) - return + return None lexical = dict(lexical_routing(candidates, queries, gold=0), queries_self_derived=self_derived) out = checked_out_path(args.out, "routing", default=str(_default_dir(args.artifact) / "routing.json")) @@ -437,6 +438,43 @@ def cmd_routing(args: argparse.Namespace) -> None: print(f"OK Saved routing measurement: {out}") print_routing_summary(payload, len(competitors), self_derived) + return payload, out + + +def _subject_name(artifact: str) -> str: + """The subject id of a routed skill: its directory name, as `catalog check` names it.""" + path = Path(str(artifact).split(":", 1)[-1]) + return path.parent.name if path.name == "SKILL.md" else path.name + + +def cmd_skill_routing(args: argparse.Namespace) -> None: + """`skill routing`, and with --json its cli_output.v1 document. Report only: the leaf issues no verdict. + + The lexical floor from self-derived (heading) queries is triage, not a routing rate, so + ``summary.queries_self_derived`` is carried next to every score. + """ + if not getattr(args, "json", False): + cmd_routing(args) + return + from dokimi import cli_output + + with cli_output.quiet_stdout(): + done = cmd_routing(args) + if done is None: # --estimate-only: an estimate, nothing measured + cli_output.emit_leaf("skill routing", None, reason="estimate only, nothing measured") + return + payload, out = done + lexical, model = payload["lexical"], payload.get("model") or {} + spent = (payload.get("cost") or {}).get("usd") + cost = {"spent_usd": spent or 0.0, "cap_usd": getattr(args, "max_usd", None), + "calls": (payload.get("cost") or {}).get("llm_calls", 0), "priced": spent is not None} + if cost["cap_usd"] is not None: + cost["remaining_usd"] = round(cost["cap_usd"] - cost["spent_usd"], 6) + cli_output.emit_leaf( + "skill routing", payload, subjects=[{"subject": _subject_name(args.artifact), "verdict": None}], + summary={"queries_self_derived": payload["queries_self_derived"], "n_candidates": payload["n_candidates"], + "lexical_recall_at_1": lexical.get("recall_at_1"), "model_recall_at_1": model.get("recall_at_1")}, + artifacts={"out": str(out)}, cost=cost) def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derived: bool, diff --git a/dokimi/dokimi/commands/repo.py b/dokimi/dokimi/commands/repo.py index eb599ae90..d720f9e85 100644 --- a/dokimi/dokimi/commands/repo.py +++ b/dokimi/dokimi/commands/repo.py @@ -109,8 +109,10 @@ def cmd_repo_validate(args: argparse.Namespace) -> None: if args.json: reason = None if not failed else (f"{len(report.errors)} error(s)" if not report.ok else f"{len(report.warnings)} warning(s) under --fail-on-warning") + findings = [cli_output.finding("error", "validate-error", str(e), manifest.path) for e in report.errors] + findings += [cli_output.finding("warn", "validate-warning", str(w), manifest.path) for w in report.warnings] cli_output.emit_leaf("repo validate", report.to_dict(), status="failed" if failed else "complete", - reason=reason) + reason=reason, subjects=[{"subject": manifest.id, "verdict": None, "findings": findings}]) else: mark = "✓" if report.ok else "✗" families = ", ".join(f"{k} {v}" for k, v in report.to_dict()["tasks"]["by_family"].items()) @@ -143,6 +145,7 @@ def _read_static(path: str) -> Any: def cmd_repo_scan(args: argparse.Namespace) -> None: """L0: check every mechanically checkable claim in the guidance against the same tree.""" + from dokimi.commands.repo_scan import scan_subjects from dokimi.artifacts import save_artifact, stamp from dokimi.repos.l0 import scan_repo from dokimi.repos.l0.typecheck import TypecheckError @@ -171,7 +174,10 @@ def cmd_repo_scan(args: argparse.Namespace) -> None: failed = bool(args.fail_on_fail and (payload["counts"]["fail"] or payload["incomplete"]) or static_errors) cli_output.emit_leaf("repo scan", stamp("repo_readiness", dict(payload)), status="failed" if failed else "complete", - reason=_scan_reason(payload, static_errors, args.fail_on_fail) if failed else None) + reason=_scan_reason(payload, static_errors, args.fail_on_fail) if failed else None, + subjects=scan_subjects(payload), + summary={"claims": dict(payload["counts"]), "files": len(payload["files"]), + "adapters": ", ".join(payload.get("adapters") or []) or "none"}) else: _print_scan_text(manifest, payload, args.out) _scan_gate(payload, args.fail_on_fail) # runs first: an invalid --static still gets the incomplete reason diff --git a/dokimi/dokimi/commands/repo_scan.py b/dokimi/dokimi/commands/repo_scan.py index c1ea880a0..c180cf1c3 100644 --- a/dokimi/dokimi/commands/repo_scan.py +++ b/dokimi/dokimi/commands/repo_scan.py @@ -153,3 +153,20 @@ def _scan_gate(payload: dict, fail_on_fail: bool) -> None: if payload["incomplete"]: print(f"✗ scan incomplete: {'; '.join(payload['incomplete_reasons'])}", file=sys.stderr) sys.exit(1) + + +def scan_subjects(payload: dict) -> list[dict]: + """One report-only subject per guidance file; each failing claim is a finding at its file and line.""" + from dokimi.cli_output import finding + + by_file: dict[str, list[dict]] = {} + for claim in payload.get("claims") or []: + if str(claim.get("status", "")).split("_", 1)[0] != "fail": + continue + file = str(claim.get("file")) + line = claim.get("line") if isinstance(claim.get("line"), int) and claim["line"] >= 1 else None + message = f"{claim.get('kind')}: `{str(claim.get('claim'))[:80]}`: {claim.get('why')}" + by_file.setdefault(file, []).append(finding("warn", str(claim.get("status")), message, file, line)) + files = [str(f.get("path", f)) if isinstance(f, dict) else str(f) for f in payload.get("files") or []] + return [{"subject": f, "verdict": None, "findings": by_file.get(f, [])} + for f in sorted(set(files) | set(by_file))] diff --git a/dokimi/dokimi/commands/report.py b/dokimi/dokimi/commands/report.py index 94805f3d4..8246a190e 100644 --- a/dokimi/dokimi/commands/report.py +++ b/dokimi/dokimi/commands/report.py @@ -1,7 +1,7 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""report subcommand group: eval, generate, model-compare.""" +"""report subcommand group: eval, generate, model-compare, summary.""" from __future__ import annotations @@ -171,6 +171,18 @@ def cmd_report_generate(args: argparse.Namespace) -> None: print(report.split("---")[0]) # Print just the summary section +def cmd_report_summary(args: argparse.Namespace) -> None: + """Merge cli_output.v1 documents into one summary. Exit 0 once written: the gate stays each leaf's own exit code.""" + from dokimi.report.summary import render, summarize + + text = render(summarize(args.inputs), args.format) + if args.out: + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + Path(args.out).write_text(text, encoding="utf-8") + else: + sys.stdout.write(text) + + def register(sub, positive_int) -> None: """Add the `report` subcommand group.""" report_p = sub.add_parser("report", help="Generate analysis reports") @@ -251,3 +263,15 @@ def register(sub, positive_int) -> None: ), ) mc_p.set_defaults(func=cmd_report_model_compare) + + # report summary + sm_p = report_sub.add_parser( + "summary", + help="Merge cli_output.v1 documents (files or directories of *.json) into one summary", + ) + sm_p.add_argument("inputs", nargs="+", metavar="PATH", + help="cli_output.v1 files, or directories whose *.json files are read") + sm_p.add_argument("--format", default="markdown", choices=["markdown", "json", "annotations"], + help="markdown (job summary, PR comment, terminal), json, or annotations (::error/::warning lines)") + sm_p.add_argument("--out", default=None, help="Write to this file instead of stdout") + sm_p.set_defaults(func=cmd_report_summary) diff --git a/dokimi/dokimi/commands/routing_parser.py b/dokimi/dokimi/commands/routing_parser.py index 142ce0744..687e2ae3b 100644 --- a/dokimi/dokimi/commands/routing_parser.py +++ b/dokimi/dokimi/commands/routing_parser.py @@ -8,7 +8,7 @@ from __future__ import annotations -from dokimi.commands.novelty import cmd_routing +from dokimi.commands.novelty import cmd_routing, cmd_skill_routing from dokimi.commands.novelty_run import add_spend_cap, stops_at_cap from dokimi.defaults import DEFAULT_ANSWER_PROVIDER from dokimi.llm import ANSWER_PROVIDERS as PROVIDERS @@ -35,7 +35,11 @@ def add_skill_routing(skill_sub, positive_int) -> None: "under another path counts once) except the artifact itself, the " "siblings a harness lists next to it. Combines with --competitors") _add_routing_options(routing_p, positive_int) - routing_p.set_defaults(all_vs_all=False, func=stops_at_cap(cmd_routing)) + routing_p.add_argument("--json", action="store_true", + help="Print cli_output.v1 on stdout (the measurement under data, report only); " + "the summary goes to stderr") + routing_p.set_defaults(all_vs_all=False, func=stops_at_cap(cmd_skill_routing), + emits_cli_output=True, cli_kind="skill routing") def add_catalog_routing(catalog_sub, positive_int) -> None: diff --git a/dokimi/dokimi/report/summary.py b/dokimi/dokimi/report/summary.py new file mode 100644 index 000000000..ed365db31 --- /dev/null +++ b/dokimi/dokimi/report/summary.py @@ -0,0 +1,177 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""One summary over any number of ``cli_output.v1`` documents (issue #749, layer 1). + +It reads envelope keys only (``kind``, ``status``, ``verdict``, ``reason``, ``error``, +``subjects``, ``cost``), so a new leaf needs no change here. It computes no verdict: each row +shows the verdict its document carries, and ``verdict: null`` is report-only. A document that is +missing, unreadable or of another schema stays a row (``not_run`` or ``invalid``); it is never +dropped and never shown as a pass. Stdlib only, and no ``GITHUB_*`` variable is read. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = "cli_output.v1" +SUMMARY_SCHEMA = "report_summary.v1" +FORMATS = ("markdown", "json", "annotations") +#: Document verdicts that count against the overall state, worst first. +_BAD = ("infra_error", "invalid", "fail", "inconclusive") + + +def _row(source: str, state: str, *, kind: str | None = None, status: str | None = None, + verdict: str | None = None, reason: str | None = None, cost: dict[str, Any] | None = None, + subjects: int = 0) -> dict[str, Any]: + return {"source": source, "state": state, "kind": kind, "status": status, "verdict": verdict, + "reason": reason, "subjects": subjects, "summary": {}, + "spent_usd": (cost or {}).get("spent_usd"), "cap_usd": (cost or {}).get("cap_usd")} + + +def expand_inputs(paths: list[str]) -> list[Path]: + """Each file as given; a directory becomes its ``*.json`` files, sorted. A missing path stays, to become ``not_run``.""" + out: list[Path] = [] + for raw in paths: + p = Path(raw) + out.extend(sorted(p.glob("*.json")) if p.is_dir() else [p]) + return out + + +def load(path: Path) -> tuple[dict[str, Any], dict[str, Any] | None]: + """The check row of one input and its document, or None when it is not a ``cli_output.v1`` document.""" + source = str(path) + if not path.exists(): + return _row(source, "not_run", reason="file not found"), None + try: + doc = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, ValueError) as exc: + return _row(source, "invalid", reason=f"unreadable: {type(exc).__name__}"), None + if not isinstance(doc, dict) or doc.get("schema_version") != SCHEMA_VERSION: + got = doc.get("schema_version") if isinstance(doc, dict) else type(doc).__name__ + return _row(source, "invalid", reason=f"not {SCHEMA_VERSION} (got {got!r})"), None + if not isinstance(doc.get("kind"), str) or not isinstance(doc.get("subjects", []), list): + return _row(source, "invalid", reason="kind or subjects malformed"), None + error = doc.get("error") if isinstance(doc.get("error"), dict) else None + reason = doc.get("reason") or (error or {}).get("message") + verdict = doc.get("verdict") + state = "failed" if doc.get("status") == "failed" else (verdict or "report_only") + cost = doc.get("cost") if isinstance(doc.get("cost"), dict) else None + row = _row(source, state, kind=doc["kind"], status=doc.get("status"), verdict=verdict, reason=reason, + cost=cost, subjects=len(doc.get("subjects") or [])) + row["summary"] = _flat(doc.get("summary")) + return row, doc + + +def _flat(summary: Any, prefix: str = "") -> dict[str, Any]: + """The scalar values of a document's ``summary``, nested keys joined by ``.``; ``dokimi_version`` left out.""" + out: dict[str, Any] = {} + for key, value in (summary.items() if isinstance(summary, dict) else ()): + name = f"{prefix}{key}" + if name == "dokimi_version": + continue + if isinstance(value, dict): + out.update(_flat(value, name + ".")) + elif value is None or isinstance(value, (str, int, float, bool)): + out[name] = value + return out + + +def _overall(checks: list[dict[str, Any]]) -> str: + states = {c["state"] for c in checks} + for bad in ("failed", "invalid", "not_run", *_BAD): + if bad in states: + return bad + if "pass" in states: + return "pass" + return "report_only" if checks else "not_run" + + +def summarize(paths: list[str]) -> dict[str, Any]: + """The merged document: overall state, one row per input, one row per subject id across inputs.""" + checks: list[dict[str, Any]] = [] + subjects: dict[str, dict[str, Any]] = {} + for path in expand_inputs(paths): + row, doc = load(path) + checks.append(row) + for s in (doc or {}).get("subjects") or []: + if isinstance(s, dict) and isinstance(s.get("subject"), str): + entry = subjects.setdefault(s["subject"], {"subject": s["subject"], "verdicts": {}, "findings": []}) + entry["verdicts"][row["kind"]] = s.get("verdict") + entry["findings"] += [{**f, "check": row["kind"]} for f in s.get("findings") or [] + if isinstance(f, dict)] + spent = [c["spent_usd"] for c in checks if isinstance(c["spent_usd"], (int, float))] + return {"schema_version": SUMMARY_SCHEMA, "overall": _overall(checks), "checks": checks, + "subjects": [subjects[k] for k in sorted(subjects)], + "spent_usd": round(sum(spent), 6) if spent else None} + + +def _cell(value: Any) -> str: + return "—" if value is None else str(value).replace("|", "\\|").replace("\n", " ") + + +def render_markdown(summary: dict[str, Any]) -> str: + lines = [f"## dokimi summary: {summary['overall']}", "", + "| Check | Source | State | Verdict | Subjects | Spent / cap USD | Summary | Reason |", + "|---|---|---|---|---|---|---|---|"] + for c in summary["checks"]: + money = "—" if c["spent_usd"] is None else f"{c['spent_usd']} / {_cell(c['cap_usd'])}" + facts = ", ".join(f"{k} {_cell(v)}" for k, v in c["summary"].items()) or None + lines.append(f"| {_cell(c['kind'])} | `{_cell(c['source'])}` | {c['state']} | {_cell(c['verdict'])} " + f"| {c['subjects']} | {money} | {_cell(facts)} | {_cell(c['reason'])} |") + if summary["subjects"]: + kinds = sorted({k for s in summary["subjects"] for k in s["verdicts"]}) + lines += ["", "| Subject | " + " | ".join(kinds) + " |", "|---|" + "---|" * len(kinds)] + for s in summary["subjects"]: + counts = {k: sum(f["check"] == k for f in s["findings"]) for k in kinds} + lines.append(f"| {_cell(s['subject'])} | " + " | ".join( + (_cell(s["verdicts"].get(k)) + (f" ({counts[k]} finding{'s' * (counts[k] != 1)})" if counts[k] else "")) + if k in s["verdicts"] else "not in run" for k in kinds) + " |") + return "\n".join(lines) + "\n" + + +def _escape(text: str) -> str: + """Workflow-command escaping of a message, so a newline or ``%`` cannot end or forge the command.""" + return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + + +def _prop(text: str) -> str: + """Workflow-command escaping of a property value (``title``, ``file``).""" + return _escape(text).replace(",", "%2C").replace(":", "%3A") + + +def render_annotations(summary: dict[str, Any]) -> str: + """One ``::error`` per check that failed or could not be read, one ``::warning`` per bad verdict, and one + line per subject finding (``::error`` for level error, ``::warning`` else) with ``file=`` and ``line=`` when known. + """ + out = [] + for c in summary["checks"]: + name = _prop(c["kind"] or c["source"]) + msg = _escape(f"{c['state']}: {c['reason'] or c['source']}") + if c["state"] in ("failed", "invalid", "not_run", "infra_error"): + out.append(f"::error title={name}::{msg}") + elif c["state"] in _BAD: + out.append(f"::warning title={name}::{msg}") + for s in summary["subjects"]: + for kind, verdict in s["verdicts"].items(): + if verdict in _BAD and not any(f["check"] == kind for f in s["findings"]): + out.append(f"::warning title={_prop(str(kind))}::{_escape(s['subject'])}: {verdict}") + for f in s["findings"]: + props = [f"title={_prop(str(f['check']) + ' ' + str(f.get('code')))}"] + if f.get("file"): + props.insert(0, f"file={_prop(str(f['file']))}") + if isinstance(f.get("line"), int): + props.insert(1, f"line={f['line']}") + level = "error" if f.get("level") == "error" else "warning" + out.append(f"::{level} {','.join(props)}::{_escape(s['subject'] + ': ' + str(f.get('message')))}") + return "".join(line + "\n" for line in out) + + +def render(summary: dict[str, Any], fmt: str) -> str: + if fmt == "json": + return json.dumps(summary, indent=2) + "\n" + if fmt == "annotations": + return render_annotations(summary) + return render_markdown(summary) diff --git a/dokimi/dokimi/schemas/cli_output.v1.json b/dokimi/dokimi/schemas/cli_output.v1.json index 9c14d6fff..df18df616 100644 --- a/dokimi/dokimi/schemas/cli_output.v1.json +++ b/dokimi/dokimi/schemas/cli_output.v1.json @@ -4,77 +4,312 @@ "title": "CLI --json output v1", "description": "The one document a leaf prints on stdout with `--json`: on success, on an input error (exit 2) and on a crash (exit 5) (dokimi.cli_output). `verdict: null` means the leaf issues no verdict; a leaf that issues one exits 0 only for `pass`. `run_id` is null when DOKIMI_RUN_ID is malformed. `summary` and `artifacts` are open beyond the keys named, and `data` is the leaf's own payload; every other level is closed. `next[].argv` is the next command as an argv array.", "type": "object", - "required": ["schema_version", "kind", "run_id", "status", "verdict", "reason", "error", "summary", "subjects", - "artifacts", "cost", "limits", "next", "data"], + "required": [ + "schema_version", + "kind", + "run_id", + "status", + "verdict", + "reason", + "error", + "summary", + "subjects", + "artifacts", + "cost", + "limits", + "next", + "data" + ], "additionalProperties": false, "$defs": { - "verdict": {"enum": ["pass", "fail", "inconclusive", "invalid", "infra_error", null]}, + "verdict": { + "enum": [ + "pass", + "fail", + "inconclusive", + "invalid", + "infra_error", + null + ] + }, "error": { - "type": ["object", "null"], - "required": ["code", "message", "retryable", "field"], + "type": [ + "object", + "null" + ], + "required": [ + "code", + "message", + "retryable", + "field" + ], "additionalProperties": false, "properties": { - "code": {"enum": ["moved", "input", "no_credentials", "infra"]}, - "message": {"type": "string", "minLength": 1}, - "retryable": {"type": "boolean"}, - "field": {"type": ["string", "null"], "description": "The flag or field the error is about."} + "code": { + "enum": [ + "moved", + "input", + "no_credentials", + "infra" + ] + }, + "message": { + "type": "string", + "minLength": 1 + }, + "retryable": { + "type": "boolean" + }, + "field": { + "type": [ + "string", + "null" + ], + "description": "The flag or field the error is about." + } } }, "subject": { "type": "object", - "required": ["subject", "verdict"], + "required": [ + "subject", + "verdict" + ], "additionalProperties": false, "properties": { - "subject": {"type": "string", "minLength": 1}, - "verdict": {"$ref": "#/$defs/verdict"} + "subject": { + "type": "string", + "minLength": 1 + }, + "verdict": { + "$ref": "#/$defs/verdict" + }, + "findings": { + "type": "array", + "description": "What a check-type leaf found on this subject; file and line when it has a location (#749).", + "items": { + "type": "object", + "required": [ + "level", + "code", + "message", + "file", + "line" + ], + "additionalProperties": false, + "properties": { + "level": { + "type": "string", + "minLength": 1, + "description": "error, warn, or the leaf's own level" + }, + "code": { + "type": "string", + "minLength": 1 + }, + "message": { + "type": "string" + }, + "file": { + "type": [ + "string", + "null" + ] + }, + "line": { + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + } + } } }, "cost": { "type": "object", - "required": ["spent_usd", "cap_usd", "remaining_usd", "estimate_kind", "calls", "priced"], + "required": [ + "spent_usd", + "cap_usd", + "remaining_usd", + "estimate_kind", + "calls", + "priced" + ], "additionalProperties": false, "properties": { - "spent_usd": {"type": "number", "minimum": 0}, - "cap_usd": {"type": ["number", "null"], "minimum": 0}, - "remaining_usd": {"type": ["number", "null"]}, - "estimate_kind": {"enum": ["upper_bound", "estimate", null]}, - "calls": {"type": "integer", "minimum": 0}, - "priced": {"type": ["boolean", "null"]} + "spent_usd": { + "type": "number", + "minimum": 0 + }, + "cap_usd": { + "type": [ + "number", + "null" + ], + "minimum": 0 + }, + "remaining_usd": { + "type": [ + "number", + "null" + ] + }, + "estimate_kind": { + "enum": [ + "upper_bound", + "estimate", + null + ] + }, + "calls": { + "type": "integer", + "minimum": 0 + }, + "priced": { + "type": [ + "boolean", + "null" + ] + } } }, "next": { "type": "object", - "required": ["argv", "why", "spends", "est_usd", "kind"], + "required": [ + "argv", + "why", + "spends", + "est_usd", + "kind" + ], "additionalProperties": false, "properties": { - "argv": {"type": "array", "minItems": 1, "items": {"type": "string"}}, - "why": {"type": "string", "minLength": 1}, - "spends": {"type": "boolean"}, - "est_usd": {"type": "number", "minimum": 0}, - "kind": {"enum": ["sequence", "alternative"]} + "argv": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "why": { + "type": "string", + "minLength": 1 + }, + "spends": { + "type": "boolean" + }, + "est_usd": { + "type": "number", + "minimum": 0 + }, + "kind": { + "enum": [ + "sequence", + "alternative" + ] + } } } }, "properties": { - "schema_version": {"const": "cli_output.v1"}, - "kind": {"type": "string", "minLength": 1, "description": "What the data is: an artifact kind such as `comparison.v1`, or a leaf's name such as `backends`."}, - "run_id": {"type": ["string", "null"], "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$", "description": "The id of the run this process writes (dokimi.artifacts.run_id); null when DOKIMI_RUN_ID is malformed."}, - "status": {"enum": ["complete", "partial", "budget_stopped", "cancelled", "failed"]}, - "verdict": {"$ref": "#/$defs/verdict"}, - "reason": {"type": ["string", "null"], "description": "Why the verdict or the status is what it is (proposal §2.6)."}, - "error": {"$ref": "#/$defs/error"}, + "schema_version": { + "const": "cli_output.v1" + }, + "kind": { + "type": "string", + "minLength": 1, + "description": "What the data is: an artifact kind such as `comparison.v1`, or a leaf's name such as `backends`." + }, + "run_id": { + "type": [ + "string", + "null" + ], + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$", + "description": "The id of the run this process writes (dokimi.artifacts.run_id); null when DOKIMI_RUN_ID is malformed." + }, + "status": { + "enum": [ + "complete", + "partial", + "budget_stopped", + "cancelled", + "failed" + ] + }, + "verdict": { + "$ref": "#/$defs/verdict" + }, + "reason": { + "type": [ + "string", + "null" + ], + "description": "Why the verdict or the status is what it is (proposal \u00a72.6)." + }, + "error": { + "$ref": "#/$defs/error" + }, "summary": { "type": "object", - "required": ["dokimi_version"], - "properties": {"dokimi_version": {"type": "string", "minLength": 1}}, + "required": [ + "dokimi_version" + ], + "properties": { + "dokimi_version": { + "type": "string", + "minLength": 1 + } + }, "description": "Leaf-specific headline values; open beyond `dokimi_version`." }, - "subjects": {"type": "array", "items": {"$ref": "#/$defs/subject"}, "description": "One entry per subject of a multi-subject run."}, - "artifacts": {"type": ["object", "null"], "description": "Paths of the files the leaf wrote."}, - "cost": {"$ref": "#/$defs/cost"}, - "limits": {"type": "array", "items": {"type": "object", "required": ["code", "text"], "additionalProperties": false, - "properties": {"code": {"type": "string"}, "text": {"type": "string"}}}}, - "next": {"type": "array", "items": {"$ref": "#/$defs/next"}}, - "data": {"description": "The leaf's own payload. Open: its shape is the leaf's, named by `kind`."} + "subjects": { + "type": "array", + "items": { + "$ref": "#/$defs/subject" + }, + "description": "One entry per subject of a multi-subject run." + }, + "artifacts": { + "type": [ + "object", + "null" + ], + "description": "Paths of the files the leaf wrote." + }, + "cost": { + "$ref": "#/$defs/cost" + }, + "limits": { + "type": "array", + "items": { + "type": "object", + "required": [ + "code", + "text" + ], + "additionalProperties": false, + "properties": { + "code": { + "type": "string" + }, + "text": { + "type": "string" + } + } + } + }, + "next": { + "type": "array", + "items": { + "$ref": "#/$defs/next" + } + }, + "data": { + "description": "The leaf's own payload. Open: its shape is the leaf's, named by `kind`." + } } } diff --git a/dokimi/tests/test_report_summary.py b/dokimi/tests/test_report_summary.py new file mode 100644 index 000000000..f0676e4f0 --- /dev/null +++ b/dokimi/tests/test_report_summary.py @@ -0,0 +1,210 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""`dokimi report summary`: one summary over cli_output.v1 documents (#749).""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +from dokimi import cli_output +from dokimi.report.summary import render, summarize + +CLI = Path(__file__).resolve().parents[1] / "cli.py" + + +def _doc(tmp: Path, name: str, kind: str, **fields) -> Path: + p = tmp / name + p.write_text(json.dumps(cli_output.envelope(kind, **fields)), encoding="utf-8") + return p + + +def test_rows_keep_each_documents_own_verdict(tmp_path): + _doc(tmp_path, "a.json", "repo scan", verdict="pass", subjects=[{"subject": "s1", "verdict": "pass"}]) + _doc(tmp_path, "b.json", "memory lint", subjects=[{"subject": "s1", "verdict": "fail"}, + {"subject": "s2", "verdict": None}]) + out = summarize([str(tmp_path)]) + states = {c["kind"]: c["state"] for c in out["checks"]} + assert states == {"repo scan": "pass", "memory lint": "report_only"} + assert out["overall"] == "pass" # a null verdict is report-only, never promoted to a fail + assert out["subjects"] == [ + {"subject": "s1", "verdicts": {"repo scan": "pass", "memory lint": "fail"}, "findings": []}, + {"subject": "s2", "verdicts": {"memory lint": None}, "findings": []}] + + +def test_missing_unreadable_and_foreign_inputs_stay_rows(tmp_path): + (tmp_path / "bad.json").write_text("{not json", encoding="utf-8") + (tmp_path / "other.json").write_text(json.dumps({"schema_version": "plan.v1"}), encoding="utf-8") + good = _doc(tmp_path, "ok.json", "repo scan", verdict="pass") + out = summarize([str(tmp_path / "bad.json"), str(tmp_path / "other.json"), str(good), + str(tmp_path / "absent.json")]) + assert [c["state"] for c in out["checks"]] == ["invalid", "invalid", "pass", "not_run"] + assert out["overall"] == "invalid" + assert summarize([str(tmp_path / "absent.json")])["overall"] == "not_run" + + +def test_failed_status_and_cost_are_shown(tmp_path): + p = tmp_path / "x.json" + p.write_text(json.dumps(cli_output.error_document("arms run", "infra", "docker gone", + cost={"spent_usd": 1.5, "cap_usd": 5.0})), encoding="utf-8") + out = summarize([str(p)]) + assert out["overall"] == "failed" and out["spent_usd"] == 1.5 + assert out["checks"][0]["reason"] == "docker gone" + md = render(out, "markdown") + assert "| arms run |" in md and "1.5 / 5.0" in md and "docker gone" in md + + +def test_annotations_escape_messages(tmp_path): + p = _doc(tmp_path, "x.json", "catalog check", verdict="fail", reason="two\nlines 100%", + subjects=[{"subject": "a::b", "verdict": "fail"}]) + lines = render(summarize([str(p)]), "annotations").splitlines() + assert lines == ["::warning title=catalog check::fail: two%0Alines 100%25", + "::warning title=catalog check::a::b: fail"] + + +def test_markdown_escapes_pipes_and_marks_absent_subjects(tmp_path): + _doc(tmp_path, "a.json", "k1", verdict="pass", reason="a|b", subjects=[{"subject": "s", "verdict": "pass"}]) + _doc(tmp_path, "b.json", "k2", verdict="pass") + md = render(summarize([str(tmp_path)]), "markdown") + assert "a\\|b" in md and "| s | pass | not in run |" not in md # k2 has no subject column + assert "| s | pass |" in md + + +def test_cli_exits_zero_on_a_failed_input_and_writes_out(tmp_path): + out = tmp_path / "r" / "summary.json" + done = subprocess.run([sys.executable, str(CLI), "report", "summary", str(tmp_path / "nope.json"), + "--format", "json", "--out", str(out)], capture_output=True, text=True) + assert done.returncode == 0, done.stderr + assert json.loads(out.read_text())["checks"][0]["state"] == "not_run" + + +# --- the leaves that opt in (#749 layer 2) and their findings in the summary --- + +import contextlib # noqa: E402 +import io # noqa: E402 +from unittest.mock import patch # noqa: E402 + +from dokimi.artifacts import schema_errors # noqa: E402 +from dokimi.cli import main # noqa: E402 + + +def _main(*argv: str) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + code = 0 + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err), patch.object(sys, "argv", ["dokimi", *argv]): + try: + main() + except SystemExit as exc: + code = int(exc.code) if isinstance(exc.code, int) else (1 if exc.code else 0) + return code, out.getvalue(), err.getvalue() + + +def _catalog(tmp: Path, broken: bool = False) -> Path: + root = tmp / "catalog" + for name in ("alpha", "beta"): + skill = root / "skills" / name + skill.mkdir(parents=True) + text = f"---\nname: {name}\ndescription: Use for {name} tasks.\n---\nRun the checks.\n" + if broken and name == "beta": + text = "---\nname: [unclosed\n---\n" + (skill / "SKILL.md").write_text(text, encoding="utf-8") + return root + + +def test_catalog_check_json_is_one_document_with_a_subject_per_skill(tmp_path): + code, out, _ = _main("catalog", "check", "--root", str(_catalog(tmp_path, broken=True)), "--kind", "skills", "--json") + doc = json.loads(out) + assert code == 1 and schema_errors("cli_output.v1", doc) == [] + assert (doc["kind"], doc["verdict"]) == ("catalog check", "fail") + subjects = {s["subject"]: s for s in doc["subjects"]} + assert subjects["alpha"]["verdict"] == "pass" + assert subjects["beta"]["verdict"] == "fail" and subjects["beta"]["findings"][0]["level"] == "error" + assert subjects["beta"]["findings"][0]["file"].endswith("SKILL.md") + + +def test_catalog_check_json_refuses_format_json(tmp_path): + code, out, _ = _main("catalog", "check", "--root", str(_catalog(tmp_path)), "--json", "--format", "json") + assert code == 2 and json.loads(out)["error"]["code"] == "input" + + +def test_catalog_eval_json_names_each_skill_and_keeps_its_receipts(tmp_path): + root = _catalog(tmp_path) + out_dir = tmp_path / "out" + code, out, _ = _main("catalog", "eval", str(root), "--tier", "static", "--out-dir", str(out_dir), "--json") + doc = json.loads(out) + assert schema_errors("cli_output.v1", doc) == [] + assert doc["kind"] == "catalog eval" and doc["verdict"] in ("pass", "fail") + assert code == (0 if doc["verdict"] == "pass" else 1) + assert {s["subject"] for s in doc["subjects"]} == {"alpha", "beta"} + assert (out_dir / "report.md").is_file() + + +def test_skill_routing_json_is_report_only_and_its_summary_goes_to_stderr(tmp_path): + def skill(name, description): + d = tmp_path / name + d.mkdir() + (d / "SKILL.md").write_text(f"---\nname: {name}\ndescription: {description}\n---\n\n# B\n\n## A section here\n", + encoding="utf-8") + return str(d) + code, out, err = _main("skill", "routing", skill("a", "profile a binary"), "--competitors", + skill("b", "tune a database"), "--queries-from-headings", + "--out", str(tmp_path / "r.json"), "--json") + doc = json.loads(out) + assert code == 0 and schema_errors("cli_output.v1", doc) == [] + assert doc["verdict"] is None and doc["subjects"] == [{"subject": "a", "verdict": None}] + assert doc["summary"]["queries_self_derived"] is True and "triage only" in err + assert doc["cost"]["spent_usd"] == 0.0 + + +def test_findings_become_located_annotations_and_counts(tmp_path): + doc = cli_output.envelope("catalog check", verdict="fail", subjects=[{"subject": "s", "verdict": "fail", "findings": [ + cli_output.finding("error", "load-error", "bad yaml", "skills/s/SKILL.md"), + cli_output.finding("warn", "w", "x", "a,b:c.md", 3)]}]) + assert schema_errors("cli_output.v1", doc) == [] + p = tmp_path / "c.json" + p.write_text(json.dumps(doc), encoding="utf-8") + summary = summarize([str(p)]) + lines = render(summary, "annotations").splitlines() + assert "::error file=skills/s/SKILL.md,title=catalog check load-error::s: bad yaml" in lines + assert "::warning file=a%2Cb%3Ac.md,line=3,title=catalog check w::s: x" in lines + assert not any(line.endswith("s: fail") for line in lines) # the findings stand in for the bare verdict + assert "| s | fail (2 findings) |" in render(summary, "markdown") + + +def test_summary_values_are_a_column(tmp_path): + _doc(tmp_path, "a.json", "repo scan", summary={"claims": {"ok": 3, "fail": 1}, "files": 2, "big": [1]}) + md = render(summarize([str(tmp_path)]), "markdown") + assert "claims.ok 3, claims.fail 1, files 2" in md and "dokimi_version" not in md and "big" not in md + + +def test_the_report_action_runs_the_renderer_and_parses_nothing(): + import yaml + + action = yaml.safe_load((CLI.parent.parent / "actions" / "report" / "action.yml").read_text(encoding="utf-8")) + runs = "\n".join(s.get("run", "") for s in action["runs"]["steps"]) + assert "python" not in runs and "jq" not in runs.replace("--jq", "") + for fmt in ("markdown", "json", "annotations"): + assert f"dokimi report summary --format {fmt}" in runs + assert '-- "$OUT"' in runs and "GITHUB_STEP_SUMMARY" in runs + assert set(action["inputs"]) == {"out-dir", "comment", "github-token", "artifact-name"} + + +def test_repo_eval_renders_validate_and_scan_with_the_renderer(): + wf = (CLI.parent.parent / ".github" / "workflows" / "repo-eval.yml").read_text(encoding="utf-8") + assert 'dokimi report summary "$RUNNER_TEMP/validate.json"' in wf + assert 'dokimi report summary "$RUNNER_TEMP/repo-scan.json"' in wf + assert '["counts"]' not in wf # no leaf field read in YAML for these two + + +def test_repo_scan_subjects_put_each_failing_claim_at_its_file_and_line(): + from dokimi.commands.repo_scan import scan_subjects + + subjects = scan_subjects({"files": ["AGENTS.md", "CLAUDE.md"], "claims": [ + {"file": "AGENTS.md", "line": 4, "status": "fail_missing", "kind": "path", "claim": "src/x.py", "why": "absent"}, + {"file": "AGENTS.md", "line": 5, "status": "ok", "kind": "path", "claim": "src", "why": None}]}) + assert [s["subject"] for s in subjects] == ["AGENTS.md", "CLAUDE.md"] + (f,) = subjects[0]["findings"] + assert (f["file"], f["line"], f["code"]) == ("AGENTS.md", 4, "fail_missing") and subjects[1]["findings"] == [] From 2137cb616a8555cf71f1c1f6506c087ca40006c6 Mon Sep 17 00:00:00 2001 From: vchamarthi Date: Tue, 17 Nov 2026 03:19:45 +0000 Subject: [PATCH 2/6] fix(report): keep every input, validate envelopes, add summary schema Fixes the review threads on report summary and its leaves: - an empty input directory stays a not_run row - malformed verdict, status, subjects or cost make an invalid row, not a TypeError - two documents of one kind keep their own subject verdicts - the subject table header is escaped like the check table - report_summary.v1 gets a bundled schema and a registry entry - skill routing --json names the loaded skill, carries selection_rate and still prints its document when the cap or a model error stops the run - catalog check --json refuses --jsonl, which would leave it no rows - actions/report finds the sticky comment of the token's own user and uploads out-dir after a failed comment step - the CLI interface snapshot lists the report_summary kind of runs list --kind Signed-off-by: vchamarthi --- actions/report/action.yml | 6 +- ...261116_120000_vchamarthi_report-summary.md | 4 +- dokimi/docs/reference/cli-interface.json | 2 +- dokimi/dokimi/artifacts.py | 5 +- dokimi/dokimi/commands/catalog.py | 10 +- dokimi/dokimi/commands/novelty.py | 38 ++++-- dokimi/dokimi/report/summary.py | 45 +++++- dokimi/dokimi/runs/kinds.py | 1 + dokimi/dokimi/schemas/report_summary.v1.json | 65 +++++++++ .../fixtures/artifacts/report_summary.v1.json | 15 ++ dokimi/tests/test_report_summary.py | 129 ++++++++++++++++++ dokimi/tests/test_runs_scan.py | 1 + dokimi/tests/test_runs_show.py | 1 + 13 files changed, 293 insertions(+), 29 deletions(-) create mode 100644 dokimi/dokimi/schemas/report_summary.v1.json create mode 100644 dokimi/tests/fixtures/artifacts/report_summary.v1.json diff --git a/actions/report/action.yml b/actions/report/action.yml index e72b71959..5129449df 100644 --- a/actions/report/action.yml +++ b/actions/report/action.yml @@ -58,15 +58,17 @@ runs: run: | marker='' { echo "$marker"; cat -- "$OUT/summary/summary.md"; } > "$OUT/summary/comment.md" + # The comment is the token's own: a user token posts as that user, the default token as the bot. + me=$(gh api user --jq .login 2>/dev/null || echo 'github-actions[bot]') id=$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ - --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | head -n1) + --jq ".[] | select(.user.login == \"$me\" and (.body | startswith(\"$marker\"))) | .id" | head -n1) if [ -n "$id" ]; then gh api -X PATCH "repos/$REPO/issues/comments/$id" -F "body=@$OUT/summary/comment.md" >/dev/null else gh api -X POST "repos/$REPO/issues/$PR/comments" -F "body=@$OUT/summary/comment.md" >/dev/null fi - name: Upload out-dir - if: ${{ inputs.artifact-name != '' }} + if: ${{ always() && inputs.artifact-name != '' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ inputs.artifact-name }} diff --git a/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md b/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md index 360b7c3bc..2c49122a5 100644 --- a/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md +++ b/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md @@ -1,6 +1,6 @@ ### Added -- **`dokimi report summary` merges `cli_output.v1` documents into one summary** (#749, the renderer). It takes files or directories of `*.json` and writes `--format markdown` (a check table and a subject table, usable as a job summary, a PR comment or terminal output), `json` (`report_summary.v1`) or `annotations` (`::error`/`::warning` lines). It reads envelope keys only, so a new leaf needs no change in it. A missing, unreadable or non-`cli_output.v1` input stays a row (`not_run` or `invalid`) and makes the overall state bad; it is never dropped or shown as a pass. It computes no verdict and exits 0 once written: the gate stays each leaf's own exit code. Stdlib only, and it reads no `GITHUB_*` variable. -- **`catalog check`, `catalog eval` and `skill routing` take `--json`** (#749, #94 F2): one `cli_output.v1` document on stdout, with one subject per skill (and per task for `catalog check`). Exit codes are unchanged. `skill routing --json` is report only (`verdict: null`) and carries `queries_self_derived` next to its scores; its human summary goes to stderr. `catalog check --json` refuses `--format json`. +- **`dokimi report summary` merges `cli_output.v1` documents into one summary** (#749, the renderer). It takes files or directories of `*.json` and writes `--format markdown` (a check table and a subject table, usable as a job summary, a PR comment or terminal output), `json` (`report_summary.v1`, with a bundled schema) or `annotations` (`::error`/`::warning` lines). It reads envelope keys only, so a new leaf needs no change in it. A missing, unreadable, malformed or non-`cli_output.v1` input, and a directory with no `*.json` file, stays a row (`not_run` or `invalid`) and makes the overall state bad; it is never dropped or shown as a pass. It computes no verdict and exits 0 once written: the gate stays each leaf's own exit code. Stdlib only, and it reads no `GITHUB_*` variable. +- **`catalog check`, `catalog eval` and `skill routing` take `--json`** (#749, #94 F2): one `cli_output.v1` document on stdout, with one subject per skill (and per task for `catalog check`). Exit codes are unchanged. `skill routing --json` is report only (`verdict: null`) and carries `queries_self_derived` next to its scores; its human summary goes to stderr. `skill routing --json` still prints its document when `--max-usd` or a model error stops the run (status `budget_stopped` or `partial`) and then exits non-zero as before; its subject is the loaded skill's name. `catalog check --json` refuses `--format json` and `--jsonl`. - **A `cli_output.v1` subject may carry `findings`** (`level`, `code`, `message`, `file`, `line`). `catalog check`, `catalog eval`, `repo validate` and `repo scan` fill them; `repo scan --json` puts each failing claim at its file and line and adds the claim counts to `summary`. `report summary --format annotations` gives each finding a `file=`/`line=` annotation, and the Markdown shows the scalar `summary` values of each document. - **`actions/report`** places the summary: it runs `dokimi report summary` over `out-dir`, appends the Markdown to the job summary, prints the annotations, keeps one sticky PR comment with `comment: sticky`, and uploads `out-dir`. No parsing and no verdict in YAML. `repo-eval.yml` renders `repo validate` and `repo scan` with the same renderer instead of two inline Python blocks. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index b04ab7833..bc00e8570 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -1560,7 +1560,7 @@ "params": [ {"key": "--help/-h", "action": "_HelpAction", "choices": null, "const": null, "default": "==SUPPRESS==", "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, {"key": "--json", "action": "_StoreTrueAction", "choices": null, "const": true, "default": false, "group_key": null, "group_required": null, "hidden": false, "is_flag": true, "kind": "option", "nargs": 0, "position": null, "required": false, "type": null}, - {"key": "--kind", "action": "_AppendAction", "choices": ["'answers'", "'arms'", "'catalog_eval'", "'checklist'", "'cli_routes'", "'cross_judge'", "'dokimi_config'", "'effective_instructions'", "'eval'", "'eval_coverage'", "'explore_runs'", "'grounding'", "'guidance_reach'", "'headroom'", "'judge_validity'", "'judge_wording'", "'leak'", "'matrix_rollup'", "'need_graph'", "'novelty'", "'pairwise'", "'persona_analysis'", "'personas'", "'plan'", "'plugin_delta'", "'question_panel'", "'question_quality'", "'questions'", "'ragas'", "'reach_matrix'", "'redteam'", "'repo_budget'", "'repo_environment'", "'repo_hits_judged'", "'repo_ledger'", "'repo_readiness'", "'report_qa'", "'result'", "'run_context'", "'security_report'", "'skill_coverage'", "'skill_discovery'", "'skill_evidence'", "'subject_scorecard'", "'task_inspect'", "'task_runs'", "'tool_preference'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, + {"key": "--kind", "action": "_AppendAction", "choices": ["'answers'", "'arms'", "'catalog_eval'", "'checklist'", "'cli_routes'", "'cross_judge'", "'dokimi_config'", "'effective_instructions'", "'eval'", "'eval_coverage'", "'explore_runs'", "'grounding'", "'guidance_reach'", "'headroom'", "'judge_validity'", "'judge_wording'", "'leak'", "'matrix_rollup'", "'need_graph'", "'novelty'", "'pairwise'", "'persona_analysis'", "'personas'", "'plan'", "'plugin_delta'", "'question_panel'", "'question_quality'", "'questions'", "'ragas'", "'reach_matrix'", "'redteam'", "'repo_budget'", "'repo_environment'", "'repo_hits_judged'", "'repo_ledger'", "'repo_readiness'", "'report_qa'", "'report_summary'", "'result'", "'run_context'", "'security_report'", "'skill_coverage'", "'skill_discovery'", "'skill_evidence'", "'subject_scorecard'", "'task_inspect'", "'task_runs'", "'tool_preference'"], "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null}, {"key": "--subject", "action": "_StoreAction", "choices": null, "const": null, "default": null, "group_key": null, "group_required": null, "hidden": false, "is_flag": false, "kind": "option", "nargs": null, "position": null, "required": false, "type": null} ], "path": "dokimi runs list", diff --git a/dokimi/dokimi/artifacts.py b/dokimi/dokimi/artifacts.py index e6c7ad5bf..325463049 100644 --- a/dokimi/dokimi/artifacts.py +++ b/dokimi/dokimi/artifacts.py @@ -103,6 +103,7 @@ "question_panel": "question_panel.v1", "question_quality": "question_quality.v1", "report_qa": "report_qa.v1", + "report_summary": "report_summary.v1", } # kind -> superseded versions still accepted on *load*, newest first. An artifact written under an @@ -116,9 +117,9 @@ #: What a run is given (question sets, whose bytes ``question_set_hash`` digests, a config, a ledger, a need -#: graph) and the CLI's route table: :func:`save_artifact` gives them no envelope (#277, #387). +#: graph), the CLI's route table and the report summary: :func:`save_artifact` gives them no envelope (#277, #387). INPUT_KINDS: frozenset[str] = frozenset({"questions", "dokimi_config", "repo_ledger", "need_graph", - "cli_routes"}) + "cli_routes", "report_summary"}) #: The envelope keys :func:`save_artifact` writes on a run kind, after ``schema_version``. ENVELOPE_KEYS: tuple[str, ...] = ("run_id", "dokimi_version") diff --git a/dokimi/dokimi/commands/catalog.py b/dokimi/dokimi/commands/catalog.py index c6def2495..d122702cd 100644 --- a/dokimi/dokimi/commands/catalog.py +++ b/dokimi/dokimi/commands/catalog.py @@ -167,9 +167,13 @@ def _scan_tasks(args: argparse.Namespace, root: Path, report: dict, as_json: boo return failed -def _refuse_json_with_format_json(args: argparse.Namespace) -> None: - if getattr(args, "json", False) and args.format == "json": +def _refuse_json_conflicts(args: argparse.Namespace) -> None: + if not getattr(args, "json", False): + return + if args.format == "json": raise InputError("--json prints cli_output.v1 with the report under data; drop --format json") + if args.jsonl: + raise InputError("--jsonl writes one row per skill to a file, so --json would carry none; drop one of them") def _check_scan_flags(args: argparse.Namespace) -> None: @@ -189,7 +193,7 @@ def _check_scan_flags(args: argparse.Namespace) -> None: if "unreadable" in (args.ignore or ()): # The whole-catalog exit reads the unfiltered findings, --per-skill the filtered rows: ignoring it splits them. raise InputError("--ignore unreadable: a SKILL.md the OS will not read always fails the scan, as a load error does") - _refuse_json_with_format_json(args) + _refuse_json_conflicts(args) if args.jsonl and args.format == "json": raise InputError("--jsonl writes one row per skill to a file; drop --format json") if args.resume and not args.jsonl: diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 33d2c19c5..4c6193147 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -379,7 +379,7 @@ def _cmd_routing_all_vs_all(args: argparse.Namespace) -> None: print_all_vs_all_summary(*summary) -def cmd_routing(args: argparse.Namespace) -> tuple[dict, Path] | None: +def cmd_routing(args: argparse.Namespace) -> tuple[dict, Path, str] | None: """Can a skill (or each skill under a root) be selected against its competitors? Lexical floor, model on top.""" from dokimi.novelty import cost if args.repeats > 1 and not args.model: @@ -433,18 +433,23 @@ def cmd_routing(args: argparse.Namespace) -> tuple[dict, Path] | None: payload["cost"] = roll_up_cost([selector]) out = save_result(payload, out, payload.get("model", {})) - stop_if_partial(payload, out, - lambda: print_routing_summary(payload, len(competitors), self_derived)) + try: + stop_if_partial(payload, out, + lambda: print_routing_summary(payload, len(competitors), self_derived)) + except SystemExit as stop: + raise RoutingStopped(stop.code, payload, out, gold.name) from None print(f"OK Saved routing measurement: {out}") print_routing_summary(payload, len(competitors), self_derived) - return payload, out + return payload, out, gold.name + +class RoutingStopped(SystemExit): + """A routing run the cap or a model error cut short: it exits as before, and carries the saved result for --json.""" -def _subject_name(artifact: str) -> str: - """The subject id of a routed skill: its directory name, as `catalog check` names it.""" - path = Path(str(artifact).split(":", 1)[-1]) - return path.parent.name if path.name == "SKILL.md" else path.name + def __init__(self, message: object, payload: dict, out: Path, skill: str) -> None: + super().__init__(message) + self.result = (payload, out, skill) def cmd_skill_routing(args: argparse.Namespace) -> None: @@ -458,23 +463,32 @@ def cmd_skill_routing(args: argparse.Namespace) -> None: return from dokimi import cli_output + stopped = None with cli_output.quiet_stdout(): - done = cmd_routing(args) + try: + done = cmd_routing(args) + except RoutingStopped as stop: # the result is saved: its document is printed, then the exit stands + done, stopped = stop.result, stop if done is None: # --estimate-only: an estimate, nothing measured cli_output.emit_leaf("skill routing", None, reason="estimate only, nothing measured") return - payload, out = done + payload, out, skill = done lexical, model = payload["lexical"], payload.get("model") or {} spent = (payload.get("cost") or {}).get("usd") cost = {"spent_usd": spent or 0.0, "cap_usd": getattr(args, "max_usd", None), "calls": (payload.get("cost") or {}).get("llm_calls", 0), "priced": spent is not None} if cost["cap_usd"] is not None: cost["remaining_usd"] = round(cost["cap_usd"] - cost["spent_usd"], 6) + partial = payload.get("partial") + status = "complete" if partial is None else ("budget_stopped" if partial["reason"] == "spend_cap" else "partial") cli_output.emit_leaf( - "skill routing", payload, subjects=[{"subject": _subject_name(args.artifact), "verdict": None}], + "skill routing", payload, status=status, reason=partial and partial["message"], + subjects=[{"subject": skill, "verdict": None}], summary={"queries_self_derived": payload["queries_self_derived"], "n_candidates": payload["n_candidates"], - "lexical_recall_at_1": lexical.get("recall_at_1"), "model_recall_at_1": model.get("recall_at_1")}, + "lexical_recall_at_1": lexical.get("recall_at_1"), "model_selection_rate": model.get("selection_rate")}, artifacts={"out": str(out)}, cost=cost) + if stopped is not None: + raise stopped def _refuse_no_queries(args: argparse.Namespace, queries: list[str], self_derived: bool, diff --git a/dokimi/dokimi/report/summary.py b/dokimi/dokimi/report/summary.py index ed365db31..2f7b65d8c 100644 --- a/dokimi/dokimi/report/summary.py +++ b/dokimi/dokimi/report/summary.py @@ -18,6 +18,8 @@ SCHEMA_VERSION = "cli_output.v1" SUMMARY_SCHEMA = "report_summary.v1" +_VERDICTS = ("pass", "fail", "inconclusive", "invalid", "infra_error") +_STATUSES = ("complete", "partial", "budget_stopped", "cancelled", "failed") FORMATS = ("markdown", "json", "annotations") #: Document verdicts that count against the overall state, worst first. _BAD = ("infra_error", "invalid", "fail", "inconclusive") @@ -32,17 +34,43 @@ def _row(source: str, state: str, *, kind: str | None = None, status: str | None def expand_inputs(paths: list[str]) -> list[Path]: - """Each file as given; a directory becomes its ``*.json`` files, sorted. A missing path stays, to become ``not_run``.""" + """Each file as given; a directory becomes its ``*.json`` files, sorted. A missing path, and a directory with no + ``*.json`` file, stay as given, to become a ``not_run`` row.""" out: list[Path] = [] for raw in paths: p = Path(raw) - out.extend(sorted(p.glob("*.json")) if p.is_dir() else [p]) + out.extend((sorted(p.glob("*.json")) or [p]) if p.is_dir() else [p]) return out +def _malformed(doc: dict[str, Any]) -> str | None: + """What is wrong with the envelope fields the summary consumes, or None when they hold their schema's types.""" + if not isinstance(doc.get("kind"), str): + return "kind" + if doc.get("status") not in _STATUSES: + return "status" + if doc.get("verdict") is not None and doc["verdict"] not in _VERDICTS: + return "verdict" + if not isinstance(doc.get("subjects", []), list): + return "subjects" + for s in doc.get("subjects") or []: + if not isinstance(s, dict) or not isinstance(s.get("subject"), str) or not s["subject"] \ + or (s.get("verdict") is not None and s["verdict"] not in _VERDICTS) \ + or not isinstance(s.get("findings", []), list): + return "subjects" + cost = doc.get("cost") + if cost is not None and (not isinstance(cost, dict) or any( + v is not None and (isinstance(v, bool) or not isinstance(v, (int, float))) + for v in (cost.get("spent_usd"), cost.get("cap_usd")))): + return "cost" + return None + + def load(path: Path) -> tuple[dict[str, Any], dict[str, Any] | None]: """The check row of one input and its document, or None when it is not a ``cli_output.v1`` document.""" source = str(path) + if path.is_dir(): + return _row(source, "not_run", reason="no *.json file in the directory"), None if not path.exists(): return _row(source, "not_run", reason="file not found"), None try: @@ -52,8 +80,9 @@ def load(path: Path) -> tuple[dict[str, Any], dict[str, Any] | None]: if not isinstance(doc, dict) or doc.get("schema_version") != SCHEMA_VERSION: got = doc.get("schema_version") if isinstance(doc, dict) else type(doc).__name__ return _row(source, "invalid", reason=f"not {SCHEMA_VERSION} (got {got!r})"), None - if not isinstance(doc.get("kind"), str) or not isinstance(doc.get("subjects", []), list): - return _row(source, "invalid", reason="kind or subjects malformed"), None + bad = _malformed(doc) + if bad: + return _row(source, "invalid", reason=f"{bad} malformed"), None error = doc.get("error") if isinstance(doc.get("error"), dict) else None reason = doc.get("reason") or (error or {}).get("message") verdict = doc.get("verdict") @@ -99,8 +128,10 @@ def summarize(paths: list[str]) -> dict[str, Any]: for s in (doc or {}).get("subjects") or []: if isinstance(s, dict) and isinstance(s.get("subject"), str): entry = subjects.setdefault(s["subject"], {"subject": s["subject"], "verdicts": {}, "findings": []}) - entry["verdicts"][row["kind"]] = s.get("verdict") - entry["findings"] += [{**f, "check": row["kind"]} for f in s.get("findings") or [] + # A second document of the same kind keeps its own column, named by its source. + check = row["kind"] if row["kind"] not in entry["verdicts"] else f"{row['kind']} ({row['source']})" + entry["verdicts"][check] = s.get("verdict") + entry["findings"] += [{**f, "check": check} for f in s.get("findings") or [] if isinstance(f, dict)] spent = [c["spent_usd"] for c in checks if isinstance(c["spent_usd"], (int, float))] return {"schema_version": SUMMARY_SCHEMA, "overall": _overall(checks), "checks": checks, @@ -123,7 +154,7 @@ def render_markdown(summary: dict[str, Any]) -> str: f"| {c['subjects']} | {money} | {_cell(facts)} | {_cell(c['reason'])} |") if summary["subjects"]: kinds = sorted({k for s in summary["subjects"] for k in s["verdicts"]}) - lines += ["", "| Subject | " + " | ".join(kinds) + " |", "|---|" + "---|" * len(kinds)] + lines += ["", "| Subject | " + " | ".join(_cell(k) for k in kinds) + " |", "|---|" + "---|" * len(kinds)] for s in summary["subjects"]: counts = {k: sum(f["check"] == k for f in s["findings"]) for k in kinds} lines.append(f"| {_cell(s['subject'])} | " + " | ".join( diff --git a/dokimi/dokimi/runs/kinds.py b/dokimi/dokimi/runs/kinds.py index abb0ec68c..08334b0d8 100644 --- a/dokimi/dokimi/runs/kinds.py +++ b/dokimi/dokimi/runs/kinds.py @@ -72,6 +72,7 @@ class KindFields: counts=("receipts",)), "task_inspect": KindFields(subject=("root",), counts=("items",)), "cli_routes": KindFields(counts=("roots", "leaves", "tombstones")), + "report_summary": KindFields(counts=("checks", "subjects")), "repo_ledger": KindFields(counts=("rules",)), "dokimi_config": KindFields(subject=("subjects[].id",), counts=("subjects", "shared")), "headroom": KindFields(counts=("items", "inputs")), diff --git a/dokimi/dokimi/schemas/report_summary.v1.json b/dokimi/dokimi/schemas/report_summary.v1.json new file mode 100644 index 000000000..26c81e174 --- /dev/null +++ b/dokimi/dokimi/schemas/report_summary.v1.json @@ -0,0 +1,65 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/intel/dokimi/schemas/report_summary.v1.json", + "title": "Report summary v1", + "description": "What `dokimi report summary --format json` prints (dokimi.report.summary): one row per input `cli_output.v1` document, in input order, and one row per subject id across them. `overall` is the worst row state; it is never a verdict the summary computed. A missing, unreadable or malformed input is a `not_run` or `invalid` row, never dropped. A subject's `verdicts` is keyed by the check kind, or ` ()` for a second document of the same kind.", + "type": "object", + "required": ["schema_version", "overall", "checks", "subjects", "spent_usd"], + "additionalProperties": false, + "$defs": { + "state": {"enum": ["pass", "fail", "inconclusive", "invalid", "infra_error", "failed", "not_run", "report_only"]}, + "verdict": {"enum": ["pass", "fail", "inconclusive", "invalid", "infra_error", null]} + }, + "properties": { + "schema_version": {"const": "report_summary.v1"}, + "overall": {"$ref": "#/$defs/state"}, + "checks": { + "type": "array", + "items": { + "type": "object", + "required": ["source", "state", "kind", "status", "verdict", "reason", "subjects", "summary", "spent_usd", "cap_usd"], + "additionalProperties": false, + "properties": { + "source": {"type": "string", "description": "The input path."}, + "state": {"$ref": "#/$defs/state"}, + "kind": {"type": ["string", "null"], "description": "The document's `kind`; null for a row with no readable document."}, + "status": {"type": ["string", "null"]}, + "verdict": {"$ref": "#/$defs/verdict"}, + "reason": {"type": ["string", "null"]}, + "subjects": {"type": "integer", "minimum": 0}, + "summary": {"type": "object", "description": "The scalar values of the document's `summary`, nested keys joined by `.`."}, + "spent_usd": {"type": ["number", "null"]}, + "cap_usd": {"type": ["number", "null"]} + } + } + }, + "subjects": { + "type": "array", + "items": { + "type": "object", + "required": ["subject", "verdicts", "findings"], + "additionalProperties": false, + "properties": { + "subject": {"type": "string", "minLength": 1}, + "verdicts": {"type": "object", "additionalProperties": {"$ref": "#/$defs/verdict"}}, + "findings": { + "type": "array", + "items": { + "type": "object", + "required": ["check", "level", "code", "message"], + "properties": { + "check": {"type": "string", "description": "The key of `verdicts` the finding belongs to."}, + "level": {"type": "string"}, + "code": {"type": "string"}, + "message": {"type": "string"}, + "file": {"type": ["string", "null"]}, + "line": {"type": ["integer", "null"]} + } + } + } + } + } + }, + "spent_usd": {"type": ["number", "null"], "description": "The sum of the rows' `spent_usd`; null when no row has a cost."} + } +} diff --git a/dokimi/tests/fixtures/artifacts/report_summary.v1.json b/dokimi/tests/fixtures/artifacts/report_summary.v1.json new file mode 100644 index 000000000..eda5a23f5 --- /dev/null +++ b/dokimi/tests/fixtures/artifacts/report_summary.v1.json @@ -0,0 +1,15 @@ +{ + "schema_version": "report_summary.v1", + "overall": "fail", + "checks": [ + {"source": "out/catalog-check.json", "state": "fail", "kind": "catalog check", "status": "complete", + "verdict": "fail", "reason": "1 subject(s) fail", "subjects": 1, "summary": {"tasks": 0}, + "spent_usd": 0.0, "cap_usd": null} + ], + "subjects": [ + {"subject": "alpha", "verdicts": {"catalog check": "fail"}, + "findings": [{"check": "catalog check", "level": "error", "code": "load-error", "message": "bad frontmatter", + "file": "skills/alpha/SKILL.md", "line": null}]} + ], + "spent_usd": 0.0 +} diff --git a/dokimi/tests/test_report_summary.py b/dokimi/tests/test_report_summary.py index f0676e4f0..4ff6b3c83 100644 --- a/dokimi/tests/test_report_summary.py +++ b/dokimi/tests/test_report_summary.py @@ -208,3 +208,132 @@ def test_repo_scan_subjects_put_each_failing_claim_at_its_file_and_line(): assert [s["subject"] for s in subjects] == ["AGENTS.md", "CLAUDE.md"] (f,) = subjects[0]["findings"] assert (f["file"], f["line"], f["code"]) == ("AGENTS.md", 4, "fail_missing") and subjects[1]["findings"] == [] + + +# --- review round: what a bad input or a second document of one kind must not hide --- + + +def test_an_empty_directory_stays_a_not_run_row(tmp_path): + good = _doc(tmp_path, "ok.json", "repo scan", verdict="pass") + empty = tmp_path / "empty-results" + empty.mkdir() + out = summarize([str(empty), str(good)]) + assert [c["state"] for c in out["checks"]] == ["not_run", "pass"] + assert out["checks"][0]["source"] == str(empty) and out["overall"] == "not_run" + + +def test_malformed_envelope_fields_are_invalid_rows_not_a_crash(tmp_path): + _doc(tmp_path, "a.json", "ok", verdict="pass") + docs = {"verdict": {"verdict": ["pass"]}, "status": {"status": "broken", "verdict": "pass"}, + "cost": {"cost": {"spent_usd": "1", "cap_usd": None}}, + "subjects": {"subjects": [{"subject": "s", "verdict": "maybe"}]}} + for field, fields in docs.items(): + p = tmp_path / f"{field}.json" + doc = cli_output.envelope("repo scan") + doc.update(fields) + p.write_text(json.dumps(doc), encoding="utf-8") + out = summarize([str(tmp_path)]) + rows = {Path(c["source"]).stem: c for c in out["checks"]} + assert rows["a"]["state"] == "pass" and out["overall"] == "invalid" + for field in docs: + assert (rows[field]["state"], rows[field]["reason"]) == ("invalid", f"{field} malformed") + + +def test_two_documents_of_one_kind_keep_each_subject_verdict(tmp_path): + _doc(tmp_path, "a.json", "memory lint", subjects=[{"subject": "s1", "verdict": "fail"}]) + _doc(tmp_path, "b.json", "memory lint", subjects=[{"subject": "s1", "verdict": "pass"}]) + out = summarize([str(tmp_path)]) + assert out["subjects"][0]["verdicts"] == {"memory lint": "fail", + f"memory lint ({tmp_path / 'b.json'})": "pass"} + assert "::warning title=memory lint::s1: fail" in render(out, "annotations") + assert f"memory lint ({tmp_path / 'b.json'})" in render(out, "markdown") + + +def test_markdown_subject_header_escapes_the_kind(tmp_path): + _doc(tmp_path, "a.json", "scan|lint", subjects=[{"subject": "s", "verdict": "pass"}]) + md = render(summarize([str(tmp_path)]), "markdown") + assert "| Subject | scan\\|lint |" in md and "| s | pass |" in md + + +def test_subject_annotation_title_escapes_its_delimiters(tmp_path): + p = _doc(tmp_path, "a.json", "scan,file=other.py,line=1", subjects=[{"subject": "s", "verdict": "fail"}]) + assert render(summarize([str(p)]), "annotations").splitlines()[-1] == \ + "::warning title=scan%2Cfile=other.py%2Cline=1::s: fail" + + +def test_the_json_summary_conforms_to_its_schema(tmp_path): + from dokimi.artifacts import SCHEMA_VERSIONS + + _doc(tmp_path, "a.json", "catalog check", verdict="fail", cost={"spent_usd": 0.5, "cap_usd": 1.0}, + subjects=[{"subject": "s", "verdict": "fail", "findings": [ + cli_output.finding("error", "x", "m", "SKILL.md", 3)]}]) + (tmp_path / "bad.json").write_text("{", encoding="utf-8") + doc = json.loads(render(summarize([str(tmp_path), str(tmp_path / "gone.json")]), "json")) + assert SCHEMA_VERSIONS["report_summary"] == doc["schema_version"] == "report_summary.v1" + assert schema_errors("report_summary.v1", doc) == [] + assert schema_errors("report_summary.v1", {**doc, "overall": "great"}) != [] + + +def test_catalog_check_json_refuses_jsonl(tmp_path): + code, out, _ = _main("catalog", "check", "--root", str(_catalog(tmp_path)), "--per-skill", "--jsonl", + str(tmp_path / "rows.jsonl"), "--json") + assert code == 2 and "--jsonl" in json.loads(out)["error"]["message"] + + +def _routed_skill(tmp_path, dirname="a-dir", name="a"): + d = tmp_path / dirname + d.mkdir() + (d / "SKILL.md").write_text(f"---\nname: {name}\ndescription: profile a binary\n---\n\n# B\n\n## A section here\n", + encoding="utf-8") + c = tmp_path / "b" + c.mkdir() + (c / "SKILL.md").write_text("---\nname: b\ndescription: tune a database\n---\n\n# B\n", encoding="utf-8") + return d, c + + +_COST = {"usd": 0.25, "llm_calls": 2, "input_tokens": 9, "output_tokens": 2, "cache_read_tokens": 0, + "cache_write_tokens": 0, "replayed_calls": 0} +_ARM = {"selection_rate": 0.5, "selection_rate_among_answered": 0.5, "n_queries": 1, "n_calls": 2, "n_abstained": 0, + "n_failed": 0, "repeats": 1, "model": "m", "noise": None} + + +def _routing_with_model(tmp_path, d, c, model_arm, artifact=None): + class Client: + pass + with patch("dokimi.novelty.cost.client", return_value=Client()), \ + patch("dokimi.novelty.client.roll_up_cost", return_value=_COST), \ + patch("dokimi.novelty.routing.model_routing", return_value=model_arm): + return _main("skill", "routing", artifact or str(d), "--competitors", str(c), "--queries-from-headings", + "--model", "m", "--max-usd", "1", "--out", str(tmp_path / "r.json"), "--json") + + +def test_skill_routing_json_names_the_loaded_skill_and_carries_the_model_rate(tmp_path, monkeypatch): + d, c = _routed_skill(tmp_path) + monkeypatch.chdir(d) + code, out, _ = _routing_with_model(tmp_path, d, c, _ARM, artifact=".") + doc = json.loads(out) + assert code == 0 and schema_errors("cli_output.v1", doc) == [] + assert doc["subjects"] == [{"subject": "a", "verdict": None}] + assert doc["summary"]["model_selection_rate"] == 0.5 + + +def test_skill_routing_json_of_a_capped_run_keeps_its_document_and_its_exit(tmp_path): + from dokimi.novelty.noise import partial_record + + d, c = _routed_skill(tmp_path) + arm = {**_ARM, "partial": partial_record("spend cap reached", 1, 2, "spend_cap")} + code, out, _ = _routing_with_model(tmp_path, d, c, arm) + doc = json.loads(out) + assert code == 1 and schema_errors("cli_output.v1", doc) == [] + assert (doc["status"], doc["reason"]) == ("budget_stopped", "spend cap reached") + assert doc["cost"]["spent_usd"] == 0.25 and doc["artifacts"]["out"] == str(tmp_path / "r.json") + + +def test_the_report_action_uploads_after_a_failed_comment_and_finds_its_own_comment(): + import yaml + + steps = yaml.safe_load((CLI.parent.parent / "actions" / "report" / "action.yml").read_text(encoding="utf-8"))["runs"]["steps"] + by_name = {s["name"]: s for s in steps} + assert "always()" in by_name["Upload out-dir"]["if"] + sticky = by_name["Sticky pull request comment"]["run"] + assert "gh api user" in sticky and '.user.login == \\"$me\\"' in sticky diff --git a/dokimi/tests/test_runs_scan.py b/dokimi/tests/test_runs_scan.py index cc3c50bb2..83d3b387e 100644 --- a/dokimi/tests/test_runs_scan.py +++ b/dokimi/tests/test_runs_scan.py @@ -52,6 +52,7 @@ "effective_instructions.v1": ({}, "toy", "m", "2026-10-31T00:00:00+00:00"), "task_inspect.v1": ({}, "tests/fixtures/task_formats", None, None), "cli_routes.v1": ({}, None, None, None), + "report_summary.v1": ({}, None, None, None), "repo_ledger.v1": ({}, None, None, None), "dokimi_config.v1": ({}, "skill/onetbb, repo", None, None), "headroom.v1": ({}, None, None, None), diff --git a/dokimi/tests/test_runs_show.py b/dokimi/tests/test_runs_show.py index 056ab06d4..4cf85a722 100644 --- a/dokimi/tests/test_runs_show.py +++ b/dokimi/tests/test_runs_show.py @@ -44,6 +44,7 @@ "subject_scorecard.v2": {"cells": 0}, "skill_evidence.v1": {"results": 2, "warnings": 0}, "cli_routes.v1": {"roots": 2, "leaves": 2, "tombstones": 1}, + "report_summary.v1": {"checks": 1, "subjects": 1}, "repo_ledger.v1": {"rules": 2}, "headroom.v1": {"items": 2, "inputs": 1}, "redteam.v1": {"violations": 2, "questions": 2}, From a10e74b4c07654f4b8f4c1fc78f5eabbfba39e87 Mon Sep 17 00:00:00 2001 From: vchamarthi Date: Tue, 17 Nov 2026 04:44:50 +0000 Subject: [PATCH 3/6] fix(report): validate findings, key columns once, refuse app tokens - summary: a non-string reason or error message, or a finding without string level, code and message, is an invalid row instead of output that breaks report_summary.v1. - summary: one column key per input document, so a subject missing from an earlier same-kind document is no longer filed under another key. - actions/report: a token other than the default whose login cannot be read is refused instead of assumed to be the Actions bot, which made the sticky comment duplicate on every run. Signed-off-by: vchamarthi --- actions/report/action.yml | 11 +++++- dokimi/dokimi/report/summary.py | 29 ++++++++++++++-- dokimi/tests/test_report_summary.py | 53 +++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/actions/report/action.yml b/actions/report/action.yml index 5129449df..54c3107b8 100644 --- a/actions/report/action.yml +++ b/actions/report/action.yml @@ -53,13 +53,22 @@ runs: env: OUT: ${{ inputs.out-dir }} GH_TOKEN: ${{ inputs.github-token }} + DEFAULT_TOKEN: ${{ github.token }} PR: ${{ github.event.pull_request.number }} REPO: ${{ github.repository }} run: | marker='' { echo "$marker"; cat -- "$OUT/summary/summary.md"; } > "$OUT/summary/comment.md" # The comment is the token's own: a user token posts as that user, the default token as the bot. - me=$(gh api user --jq .login 2>/dev/null || echo 'github-actions[bot]') + # Any other token whose login cannot be read (an app installation token) is refused, not guessed. + if [ "$GH_TOKEN" = "$DEFAULT_TOKEN" ]; then + me='github-actions[bot]' + else + me=$(gh api user --jq .login 2>/dev/null) || { + echo "::error title=dokimi report::github-token has no readable user login (an app installation token?); use the default token or a user token" + exit 1 + } + fi id=$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ --jq ".[] | select(.user.login == \"$me\" and (.body | startswith(\"$marker\"))) | .id" | head -n1) if [ -n "$id" ]; then diff --git a/dokimi/dokimi/report/summary.py b/dokimi/dokimi/report/summary.py index 2f7b65d8c..46c4f64ea 100644 --- a/dokimi/dokimi/report/summary.py +++ b/dokimi/dokimi/report/summary.py @@ -43,6 +43,18 @@ def expand_inputs(paths: list[str]) -> list[Path]: return out +def _str_or_none(value: Any) -> bool: + return value is None or isinstance(value, str) + + +def _finding_ok(f: Any) -> bool: + """A finding holds the types ``report_summary.v1`` states: string level, code and message, and an optional + string file and integer line.""" + return (isinstance(f, dict) and all(isinstance(f.get(k), str) for k in ("level", "code", "message")) + and _str_or_none(f.get("file")) and (f.get("line") is None or ( + isinstance(f["line"], int) and not isinstance(f["line"], bool)))) + + def _malformed(doc: dict[str, Any]) -> str | None: """What is wrong with the envelope fields the summary consumes, or None when they hold their schema's types.""" if not isinstance(doc.get("kind"), str): @@ -56,8 +68,12 @@ def _malformed(doc: dict[str, Any]) -> str | None: for s in doc.get("subjects") or []: if not isinstance(s, dict) or not isinstance(s.get("subject"), str) or not s["subject"] \ or (s.get("verdict") is not None and s["verdict"] not in _VERDICTS) \ - or not isinstance(s.get("findings", []), list): + or not isinstance(s.get("findings", []), list) \ + or not all(_finding_ok(f) for f in s.get("findings") or []): return "subjects" + error = doc.get("error") + if not _str_or_none(doc.get("reason")) or (isinstance(error, dict) and not _str_or_none(error.get("message"))): + return "reason" cost = doc.get("cost") if cost is not None and (not isinstance(cost, dict) or any( v is not None and (isinstance(v, bool) or not isinstance(v, (int, float))) @@ -122,14 +138,21 @@ def summarize(paths: list[str]) -> dict[str, Any]: """The merged document: overall state, one row per input, one row per subject id across inputs.""" checks: list[dict[str, Any]] = [] subjects: dict[str, dict[str, Any]] = {} + used: set[str] = set() for path in expand_inputs(paths): row, doc = load(path) checks.append(row) + # One column key per document: its kind, or the kind and source when an earlier document took the kind. + check = row["kind"] + if (doc or {}).get("subjects"): + n = 1 + while check in used: + n += 1 + check = f"{row['kind']} ({row['source']})" + (f" #{n}" if n > 2 else "") + used.add(check) for s in (doc or {}).get("subjects") or []: if isinstance(s, dict) and isinstance(s.get("subject"), str): entry = subjects.setdefault(s["subject"], {"subject": s["subject"], "verdicts": {}, "findings": []}) - # A second document of the same kind keeps its own column, named by its source. - check = row["kind"] if row["kind"] not in entry["verdicts"] else f"{row['kind']} ({row['source']})" entry["verdicts"][check] = s.get("verdict") entry["findings"] += [{**f, "check": check} for f in s.get("findings") or [] if isinstance(f, dict)] diff --git a/dokimi/tests/test_report_summary.py b/dokimi/tests/test_report_summary.py index 4ff6b3c83..1740088cc 100644 --- a/dokimi/tests/test_report_summary.py +++ b/dokimi/tests/test_report_summary.py @@ -249,6 +249,30 @@ def test_two_documents_of_one_kind_keep_each_subject_verdict(tmp_path): assert f"memory lint ({tmp_path / 'b.json'})" in render(out, "markdown") +def test_same_kind_documents_with_different_subjects_keep_one_key_each(tmp_path): + _doc(tmp_path, "a.json", "memory lint", subjects=[{"subject": "s1", "verdict": "fail"}]) + _doc(tmp_path, "b.json", "memory lint", subjects=[{"subject": "s1", "verdict": "pass"}, + {"subject": "s2", "verdict": "pass"}]) + out = summarize([str(tmp_path), str(tmp_path / "b.json")]) + b = f"memory lint ({tmp_path / 'b.json'})" + by = {s["subject"]: s["verdicts"] for s in out["subjects"]} + assert by["s1"] == {"memory lint": "fail", b: "pass", f"{b} #3": "pass"} + assert by["s2"] == {b: "pass", f"{b} #3": "pass"} + assert "not in run" in render(out, "markdown") # s2 is absent from a.json only + + +def test_reason_and_findings_of_the_wrong_type_are_invalid_rows(tmp_path): + bad = {"reason": {"reason": 123}, "error": {"error": {"message": 5}}, + "empty-finding": {"subjects": [{"subject": "s", "verdict": "pass", "findings": [{}]}]}} + for name, fields in bad.items(): + doc = cli_output.envelope("repo scan") + doc.update(fields) + (tmp_path / f"{name}.json").write_text(json.dumps(doc), encoding="utf-8") + out = summarize([str(tmp_path)]) + assert {Path(c["source"]).stem: c["state"] for c in out["checks"]} == {k: "invalid" for k in bad} + assert not out["subjects"] + + def test_markdown_subject_header_escapes_the_kind(tmp_path): _doc(tmp_path, "a.json", "scan|lint", subjects=[{"subject": "s", "verdict": "pass"}]) md = render(summarize([str(tmp_path)]), "markdown") @@ -337,3 +361,32 @@ def test_the_report_action_uploads_after_a_failed_comment_and_finds_its_own_comm assert "always()" in by_name["Upload out-dir"]["if"] sticky = by_name["Sticky pull request comment"]["run"] assert "gh api user" in sticky and '.user.login == \\"$me\\"' in sticky + + +def _run_sticky(tmp_path, token, default_token): + """The sticky step's script against a fake ``gh`` that has no ``/user`` and logs its calls.""" + import yaml + + steps = yaml.safe_load((CLI.parent.parent / "actions" / "report" / "action.yml").read_text(encoding="utf-8"))["runs"]["steps"] + script = next(s for s in steps if s["name"] == "Sticky pull request comment")["run"] + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + gh = bin_dir / "gh" + gh.write_text('#!/bin/sh\necho "$@" >> "$GH_LOG"\n[ "$2" = user ] && exit 1\nexit 0\n', encoding="utf-8") + gh.chmod(0o755) + (tmp_path / "summary").mkdir() + (tmp_path / "summary" / "summary.md").write_text("x\n", encoding="utf-8") + env = {"PATH": f"{bin_dir}:/usr/bin:/bin", "OUT": str(tmp_path), "GH_TOKEN": token, "DEFAULT_TOKEN": default_token, + "PR": "1", "REPO": "o/r", "GH_LOG": str(tmp_path / "gh.log")} + done = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True, check=False) + return done, (tmp_path / "gh.log").read_text(encoding="utf-8") if (tmp_path / "gh.log").exists() else "" + + +def test_sticky_comment_refuses_a_token_whose_login_is_unreadable(tmp_path): + done, log = _run_sticky(tmp_path, "app-token", "default-token") + assert done.returncode == 1 and "POST" not in log and "app installation" in done.stdout + + +def test_sticky_comment_posts_with_the_default_token(tmp_path): + done, log = _run_sticky(tmp_path, "default-token", "default-token") + assert done.returncode == 0 and "POST" in log and "api user" not in log From e2387d88ee910e37cc7e709457e790698d0488b0 Mon Sep 17 00:00:00 2001 From: vchamarthi Date: Tue, 17 Nov 2026 15:57:48 +0000 Subject: [PATCH 4/6] fix(report): stopped runs, unpriced spend, eval scope, comment limits - A document with status budget_stopped, partial or cancelled is that state in the summary, not a pass, and gets a warning annotation. - An unpriced spend prints as unpriced and leaves the total empty; skill routing --json gives priced null for no calls and no remainder for an unpriced run. - catalog eval --json gives a skill outside --changed-from verdict null and no findings, and points findings at SKILL.md under the given root. - The Markdown lists each finding (first 100) and escapes <, > and \r. - actions/report cuts the sticky comment below 65,536 characters and warns instead of failing when it cannot be posted; out-dir documents that its top-level *.json must be cli_output.v1. Signed-off-by: vchamarthi --- actions/report/action.yml | 19 ++- actions/setup/action.yml | 3 +- ...261116_120000_vchamarthi_report-summary.md | 6 +- dokimi/dokimi/commands/catalog_eval.py | 21 ++-- dokimi/dokimi/commands/novelty.py | 7 +- dokimi/dokimi/report/summary.py | 42 +++++-- dokimi/dokimi/schemas/report_summary.v1.json | 9 +- .../fixtures/artifacts/report_summary.v1.json | 2 +- dokimi/tests/test_report_summary.py | 112 +++++++++++++++++- 9 files changed, 186 insertions(+), 35 deletions(-) diff --git a/actions/report/action.yml b/actions/report/action.yml index 54c3107b8..fb8850f50 100644 --- a/actions/report/action.yml +++ b/actions/report/action.yml @@ -14,7 +14,9 @@ inputs: out-dir: description: >- Directory holding the cli_output.v1 documents (`*.json`, not recursive), usually - the out-dir output of actions/setup. The rendered summary goes to its `summary/` + the out-dir output of actions/setup. Every top-level `*.json` there must be a + cli_output.v1 document, else it is an `invalid` row; other files (a receipt, a + `--out` result) go in a subdirectory. The rendered summary goes to its `summary/` subdirectory. required: true comment: @@ -58,7 +60,11 @@ runs: REPO: ${{ github.repository }} run: | marker='' - { echo "$marker"; cat -- "$OUT/summary/summary.md"; } > "$OUT/summary/comment.md" + # A comment body is capped at 65,536 characters: cut the summary below it and point to the job summary. + { echo "$marker"; head -c 60000 -- "$OUT/summary/summary.md" + if [ "$(wc -c < "$OUT/summary/summary.md")" -gt 60000 ]; then + printf '\n\n_Summary cut here; the full one is in the job summary._\n' + fi; } > "$OUT/summary/comment.md" # The comment is the token's own: a user token posts as that user, the default token as the bot. # Any other token whose login cannot be read (an app installation token) is refused, not guessed. if [ "$GH_TOKEN" = "$DEFAULT_TOKEN" ]; then @@ -70,11 +76,14 @@ runs: } fi id=$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ - --jq ".[] | select(.user.login == \"$me\" and (.body | startswith(\"$marker\"))) | .id" | head -n1) + --jq ".[] | select(.user.login == \"$me\" and (.body | startswith(\"$marker\"))) | .id" | head -n1) || id= + # A token that cannot write (a pull request from a fork) must not turn a green job red. if [ -n "$id" ]; then - gh api -X PATCH "repos/$REPO/issues/comments/$id" -F "body=@$OUT/summary/comment.md" >/dev/null + gh api -X PATCH "repos/$REPO/issues/comments/$id" -F "body=@$OUT/summary/comment.md" >/dev/null || + echo "::warning title=dokimi report::the sticky comment could not be updated; the summary is in the job summary" else - gh api -X POST "repos/$REPO/issues/$PR/comments" -F "body=@$OUT/summary/comment.md" >/dev/null + gh api -X POST "repos/$REPO/issues/$PR/comments" -F "body=@$OUT/summary/comment.md" >/dev/null || + echo "::warning title=dokimi report::the sticky comment could not be posted; the summary is in the job summary" fi - name: Upload out-dir if: ${{ always() && inputs.artifact-name != '' }} diff --git a/actions/setup/action.yml b/actions/setup/action.yml index b2bf862b0..4c4857e09 100644 --- a/actions/setup/action.yml +++ b/actions/setup/action.yml @@ -25,7 +25,8 @@ inputs: description: >- Directory for the run output, created by this action; empty means `$RUNNER_TEMP/dokimi-out`. The caller writes its output under it, and uploads the - whole directory as an artifact. + whole directory as an artifact. When actions/report summarizes it, its top-level + `*.json` files must be cli_output.v1 documents; other files go in subdirectories. required: false default: '' diff --git a/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md b/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md index 2c49122a5..940747a7f 100644 --- a/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md +++ b/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md @@ -1,6 +1,6 @@ ### Added -- **`dokimi report summary` merges `cli_output.v1` documents into one summary** (#749, the renderer). It takes files or directories of `*.json` and writes `--format markdown` (a check table and a subject table, usable as a job summary, a PR comment or terminal output), `json` (`report_summary.v1`, with a bundled schema) or `annotations` (`::error`/`::warning` lines). It reads envelope keys only, so a new leaf needs no change in it. A missing, unreadable, malformed or non-`cli_output.v1` input, and a directory with no `*.json` file, stays a row (`not_run` or `invalid`) and makes the overall state bad; it is never dropped or shown as a pass. It computes no verdict and exits 0 once written: the gate stays each leaf's own exit code. Stdlib only, and it reads no `GITHUB_*` variable. -- **`catalog check`, `catalog eval` and `skill routing` take `--json`** (#749, #94 F2): one `cli_output.v1` document on stdout, with one subject per skill (and per task for `catalog check`). Exit codes are unchanged. `skill routing --json` is report only (`verdict: null`) and carries `queries_self_derived` next to its scores; its human summary goes to stderr. `skill routing --json` still prints its document when `--max-usd` or a model error stops the run (status `budget_stopped` or `partial`) and then exits non-zero as before; its subject is the loaded skill's name. `catalog check --json` refuses `--format json` and `--jsonl`. +- **`dokimi report summary` merges `cli_output.v1` documents into one summary** (#749, the renderer). It takes files or directories of `*.json` and writes `--format markdown` (a check table and a subject table, usable as a job summary, a PR comment or terminal output), `json` (`report_summary.v1`, with a bundled schema) or `annotations` (`::error`/`::warning` lines). It reads envelope keys only, so a new leaf needs no change in it. A missing, unreadable, malformed or non-`cli_output.v1` input, and a directory with no `*.json` file, stays a row (`not_run` or `invalid`) and makes the overall state bad; it is never dropped or shown as a pass. A document whose status is `budget_stopped`, `partial` or `cancelled` is that state, not a pass, and gets a `::warning`; an unpriced spend shows as `unpriced` and leaves the total empty; the Markdown lists each finding (file:line, code, message, the first 100). It computes no verdict and exits 0 once written: the gate stays each leaf's own exit code. Stdlib only, and it reads no `GITHUB_*` variable. +- **`catalog check`, `catalog eval` and `skill routing` take `--json`** (#749, #94 F2): one `cli_output.v1` document on stdout, with one subject per skill (and per task for `catalog check`). Exit codes are unchanged. `catalog eval --json` leaves a skill outside `--changed-from` unjudged (`verdict: null`, no findings) and points a finding at its `SKILL.md` under the given root. `skill routing --json` is report only (`verdict: null`) and carries `queries_self_derived` next to its scores; its human summary goes to stderr. `skill routing --json` still prints its document when `--max-usd` or a model error stops the run (status `budget_stopped` or `partial`) and then exits non-zero as before; its subject is the loaded skill's name. `catalog check --json` refuses `--format json` and `--jsonl`. - **A `cli_output.v1` subject may carry `findings`** (`level`, `code`, `message`, `file`, `line`). `catalog check`, `catalog eval`, `repo validate` and `repo scan` fill them; `repo scan --json` puts each failing claim at its file and line and adds the claim counts to `summary`. `report summary --format annotations` gives each finding a `file=`/`line=` annotation, and the Markdown shows the scalar `summary` values of each document. -- **`actions/report`** places the summary: it runs `dokimi report summary` over `out-dir`, appends the Markdown to the job summary, prints the annotations, keeps one sticky PR comment with `comment: sticky`, and uploads `out-dir`. No parsing and no verdict in YAML. `repo-eval.yml` renders `repo validate` and `repo scan` with the same renderer instead of two inline Python blocks. +- **`actions/report`** places the summary: it runs `dokimi report summary` over `out-dir`, appends the Markdown to the job summary, prints the annotations, keeps one sticky PR comment with `comment: sticky`, cut below the comment limit and only a warning when it cannot be posted, and uploads `out-dir`. No parsing and no verdict in YAML. `repo-eval.yml` renders `repo validate` and `repo scan` with the same renderer instead of two inline Python blocks. diff --git a/dokimi/dokimi/commands/catalog_eval.py b/dokimi/dokimi/commands/catalog_eval.py index be7a4883a..36a4eaf46 100644 --- a/dokimi/dokimi/commands/catalog_eval.py +++ b/dokimi/dokimi/commands/catalog_eval.py @@ -196,27 +196,34 @@ def cmd_catalog_eval(args: argparse.Namespace) -> None: def _finish(args: argparse.Namespace, report: dict[str, Any], out: Path) -> None: """The one line, or with --json the document; exit 1 unless the checks pass.""" if getattr(args, "json", False): - _emit(report, out) + _emit(report, out, args.catalog_root) else: print(f"Static catalog checks: {report['status']}; receipts: {out}") if report["status"] != "pass": raise SystemExit(1) -def _emit(report: dict[str, Any], out: Path) -> None: +def _emit(report: dict[str, Any], out: Path, root: Path) -> None: """The cli_output.v1 document of `catalog eval --json`: one subject per skill with its static findings. ``data`` holds the per-skill evaluation state (``no_tasks``, ``not_measured``, ``checks_only``) and the - findings that belong to no skill; the full report stays in the receipts under ``--out-dir``. + findings that belong to no skill; the full report stays in the receipts under ``--out-dir``. A finding's file is + under ``root`` as given, and a skill directory becomes its ``SKILL.md``. With ``--changed-from``, a skill outside + the selection was not evaluated: its verdict is null and it is left out of ``reason``. """ from dokimi import cli_output from dokimi.external.catalog_eval import shared_problems def finding(item: dict[str, Any]) -> dict[str, Any]: - return cli_output.finding(item["level"], item["code"], item["message"], item.get("path")) - - subjects = [{"subject": row["name"], "verdict": row["checks"]["status"], - "findings": [finding(item) for item in row["checks"]["findings"]]} for row in report["skills"]] + path = root / item["path"] if item.get("path") else None + if path is not None and path.is_dir(): + path = path / "SKILL.md" + return cli_output.finding(item["level"], item["code"], item["message"], path and path.as_posix()) + + selected = set(report["selection"]["selected"]) + subjects = [{"subject": row["name"], "verdict": row["checks"]["status"] if row["name"] in selected else None, + "findings": [finding(item) for item in row["checks"]["findings"]] if row["name"] in selected else []} + for row in report["skills"]] shared = shared_problems(report["problems"], report["skills"]) evaluation = {row["name"]: row["evaluation"] for row in report["skills"]} states: dict[str, int] = {} diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index 4c6193147..b913a919b 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -475,9 +475,10 @@ def cmd_skill_routing(args: argparse.Namespace) -> None: payload, out, skill = done lexical, model = payload["lexical"], payload.get("model") or {} spent = (payload.get("cost") or {}).get("usd") - cost = {"spent_usd": spent or 0.0, "cap_usd": getattr(args, "max_usd", None), - "calls": (payload.get("cost") or {}).get("llm_calls", 0), "priced": spent is not None} - if cost["cap_usd"] is not None: + calls = (payload.get("cost") or {}).get("llm_calls", 0) + cost = {"spent_usd": spent or 0.0, "cap_usd": getattr(args, "max_usd", None), "calls": calls, + "priced": None if not calls else spent is not None} + if cost["cap_usd"] is not None and cost["priced"] is not False: # an unpriced spend leaves no remainder to state cost["remaining_usd"] = round(cost["cap_usd"] - cost["spent_usd"], 6) partial = payload.get("partial") status = "complete" if partial is None else ("budget_stopped" if partial["reason"] == "spend_cap" else "partial") diff --git a/dokimi/dokimi/report/summary.py b/dokimi/dokimi/report/summary.py index 46c4f64ea..f5f70cd9f 100644 --- a/dokimi/dokimi/report/summary.py +++ b/dokimi/dokimi/report/summary.py @@ -23,6 +23,10 @@ FORMATS = ("markdown", "json", "annotations") #: Document verdicts that count against the overall state, worst first. _BAD = ("infra_error", "invalid", "fail", "inconclusive") +#: Document statuses that end short of complete; a row in one is not a pass. +_STOPPED = ("budget_stopped", "partial", "cancelled") +#: Findings listed under the subject table of the Markdown; the rest stay in the JSON and the annotations. +MAX_LISTED = 100 def _row(source: str, state: str, *, kind: str | None = None, status: str | None = None, @@ -30,7 +34,8 @@ def _row(source: str, state: str, *, kind: str | None = None, status: str | None subjects: int = 0) -> dict[str, Any]: return {"source": source, "state": state, "kind": kind, "status": status, "verdict": verdict, "reason": reason, "subjects": subjects, "summary": {}, - "spent_usd": (cost or {}).get("spent_usd"), "cap_usd": (cost or {}).get("cap_usd")} + "spent_usd": (cost or {}).get("spent_usd"), "cap_usd": (cost or {}).get("cap_usd"), + "priced": (cost or {}).get("priced")} def expand_inputs(paths: list[str]) -> list[Path]: @@ -102,7 +107,15 @@ def load(path: Path) -> tuple[dict[str, Any], dict[str, Any] | None]: error = doc.get("error") if isinstance(doc.get("error"), dict) else None reason = doc.get("reason") or (error or {}).get("message") verdict = doc.get("verdict") - state = "failed" if doc.get("status") == "failed" else (verdict or "report_only") + status = doc.get("status") + if status == "failed": + state = "failed" + elif verdict in _BAD: + state = verdict + elif status in _STOPPED: # a run the cap or an error cut short is not a pass, whatever verdict it carries + state = status + else: + state = verdict or "report_only" cost = doc.get("cost") if isinstance(doc.get("cost"), dict) else None row = _row(source, state, kind=doc["kind"], status=doc.get("status"), verdict=verdict, reason=reason, cost=cost, subjects=len(doc.get("subjects") or [])) @@ -126,7 +139,7 @@ def _flat(summary: Any, prefix: str = "") -> dict[str, Any]: def _overall(checks: list[dict[str, Any]]) -> str: states = {c["state"] for c in checks} - for bad in ("failed", "invalid", "not_run", *_BAD): + for bad in ("failed", "invalid", "not_run", *_BAD, *_STOPPED): if bad in states: return bad if "pass" in states: @@ -157,13 +170,18 @@ def summarize(paths: list[str]) -> dict[str, Any]: entry["findings"] += [{**f, "check": check} for f in s.get("findings") or [] if isinstance(f, dict)] spent = [c["spent_usd"] for c in checks if isinstance(c["spent_usd"], (int, float))] + unpriced = any(c["priced"] is False for c in checks) return {"schema_version": SUMMARY_SCHEMA, "overall": _overall(checks), "checks": checks, "subjects": [subjects[k] for k in sorted(subjects)], - "spent_usd": round(sum(spent), 6) if spent else None} + "spent_usd": round(sum(spent), 6) if spent and not unpriced else None} def _cell(value: Any) -> str: - return "—" if value is None else str(value).replace("|", "\\|").replace("\n", " ") + """A table cell: ``|`` and line breaks cannot split it, ``<`` and ``>`` cannot open HTML.""" + if value is None: + return "—" + return (str(value).replace("|", "\\|").replace("\n", " ").replace("\r", " ") + .replace("<", "<").replace(">", ">")) def render_markdown(summary: dict[str, Any]) -> str: @@ -171,7 +189,8 @@ def render_markdown(summary: dict[str, Any]) -> str: "| Check | Source | State | Verdict | Subjects | Spent / cap USD | Summary | Reason |", "|---|---|---|---|---|---|---|---|"] for c in summary["checks"]: - money = "—" if c["spent_usd"] is None else f"{c['spent_usd']} / {_cell(c['cap_usd'])}" + money = "—" if c["spent_usd"] is None else ( + f"unpriced / {_cell(c['cap_usd'])}" if c["priced"] is False else f"{c['spent_usd']} / {_cell(c['cap_usd'])}") facts = ", ".join(f"{k} {_cell(v)}" for k, v in c["summary"].items()) or None lines.append(f"| {_cell(c['kind'])} | `{_cell(c['source'])}` | {c['state']} | {_cell(c['verdict'])} " f"| {c['subjects']} | {money} | {_cell(facts)} | {_cell(c['reason'])} |") @@ -183,6 +202,15 @@ def render_markdown(summary: dict[str, Any]) -> str: lines.append(f"| {_cell(s['subject'])} | " + " | ".join( (_cell(s["verdicts"].get(k)) + (f" ({counts[k]} finding{'s' * (counts[k] != 1)})" if counts[k] else "")) if k in s["verdicts"] else "not in run" for k in kinds) + " |") + listed = [(s["subject"], f) for s in summary["subjects"] for f in s["findings"]] + if listed: + lines += ["", "| Subject | Check | Level | Location | Code | Message |", "|---|---|---|---|---|---|"] + for subject, f in listed[:MAX_LISTED]: + where = f.get("file") and f"{f['file']}:{f['line']}" if f.get("line") else f.get("file") + lines.append(f"| {_cell(subject)} | {_cell(f['check'])} | {_cell(f.get('level'))} | {_cell(where)} " + f"| {_cell(f.get('code'))} | {_cell(f.get('message'))} |") + if len(listed) > MAX_LISTED: + lines += ["", f"{len(listed) - MAX_LISTED} more findings in the JSON summary and the annotations"] return "\n".join(lines) + "\n" @@ -206,7 +234,7 @@ def render_annotations(summary: dict[str, Any]) -> str: msg = _escape(f"{c['state']}: {c['reason'] or c['source']}") if c["state"] in ("failed", "invalid", "not_run", "infra_error"): out.append(f"::error title={name}::{msg}") - elif c["state"] in _BAD: + elif c["state"] in _BAD or c["state"] in _STOPPED: out.append(f"::warning title={name}::{msg}") for s in summary["subjects"]: for kind, verdict in s["verdicts"].items(): diff --git a/dokimi/dokimi/schemas/report_summary.v1.json b/dokimi/dokimi/schemas/report_summary.v1.json index 26c81e174..4df965ad8 100644 --- a/dokimi/dokimi/schemas/report_summary.v1.json +++ b/dokimi/dokimi/schemas/report_summary.v1.json @@ -7,7 +7,7 @@ "required": ["schema_version", "overall", "checks", "subjects", "spent_usd"], "additionalProperties": false, "$defs": { - "state": {"enum": ["pass", "fail", "inconclusive", "invalid", "infra_error", "failed", "not_run", "report_only"]}, + "state": {"enum": ["pass", "fail", "inconclusive", "invalid", "infra_error", "failed", "not_run", "report_only", "budget_stopped", "partial", "cancelled"]}, "verdict": {"enum": ["pass", "fail", "inconclusive", "invalid", "infra_error", null]} }, "properties": { @@ -17,7 +17,7 @@ "type": "array", "items": { "type": "object", - "required": ["source", "state", "kind", "status", "verdict", "reason", "subjects", "summary", "spent_usd", "cap_usd"], + "required": ["source", "state", "kind", "status", "verdict", "reason", "subjects", "summary", "spent_usd", "cap_usd", "priced"], "additionalProperties": false, "properties": { "source": {"type": "string", "description": "The input path."}, @@ -29,7 +29,8 @@ "subjects": {"type": "integer", "minimum": 0}, "summary": {"type": "object", "description": "The scalar values of the document's `summary`, nested keys joined by `.`."}, "spent_usd": {"type": ["number", "null"]}, - "cap_usd": {"type": ["number", "null"]} + "cap_usd": {"type": ["number", "null"]}, + "priced": {"type": ["boolean", "null"], "description": "The document's `cost.priced`; false means `spent_usd` is not a real price."} } } }, @@ -60,6 +61,6 @@ } } }, - "spent_usd": {"type": ["number", "null"], "description": "The sum of the rows' `spent_usd`; null when no row has a cost."} + "spent_usd": {"type": ["number", "null"], "description": "The sum of the rows' `spent_usd`; null when no row has a cost or any row is unpriced."} } } diff --git a/dokimi/tests/fixtures/artifacts/report_summary.v1.json b/dokimi/tests/fixtures/artifacts/report_summary.v1.json index eda5a23f5..01edfc61d 100644 --- a/dokimi/tests/fixtures/artifacts/report_summary.v1.json +++ b/dokimi/tests/fixtures/artifacts/report_summary.v1.json @@ -4,7 +4,7 @@ "checks": [ {"source": "out/catalog-check.json", "state": "fail", "kind": "catalog check", "status": "complete", "verdict": "fail", "reason": "1 subject(s) fail", "subjects": 1, "summary": {"tasks": 0}, - "spent_usd": 0.0, "cap_usd": null} + "spent_usd": 0.0, "cap_usd": null, "priced": null} ], "subjects": [ {"subject": "alpha", "verdicts": {"catalog check": "fail"}, diff --git a/dokimi/tests/test_report_summary.py b/dokimi/tests/test_report_summary.py index 1740088cc..e34ecb7ba 100644 --- a/dokimi/tests/test_report_summary.py +++ b/dokimi/tests/test_report_summary.py @@ -363,7 +363,7 @@ def test_the_report_action_uploads_after_a_failed_comment_and_finds_its_own_comm assert "gh api user" in sticky and '.user.login == \\"$me\\"' in sticky -def _run_sticky(tmp_path, token, default_token): +def _run_sticky(tmp_path, token, default_token, body="x\n", write_fails=False): """The sticky step's script against a fake ``gh`` that has no ``/user`` and logs its calls.""" import yaml @@ -372,12 +372,14 @@ def _run_sticky(tmp_path, token, default_token): bin_dir = tmp_path / "bin" bin_dir.mkdir() gh = bin_dir / "gh" - gh.write_text('#!/bin/sh\necho "$@" >> "$GH_LOG"\n[ "$2" = user ] && exit 1\nexit 0\n', encoding="utf-8") + gh.write_text('#!/bin/sh\necho "$@" >> "$GH_LOG"\n[ "$2" = user ] && exit 1\n' + '[ -n "$GH_WRITE_FAILS" ] && [ "$2" = -X ] && exit 1\nexit 0\n', encoding="utf-8") gh.chmod(0o755) (tmp_path / "summary").mkdir() - (tmp_path / "summary" / "summary.md").write_text("x\n", encoding="utf-8") + (tmp_path / "summary" / "summary.md").write_text(body, encoding="utf-8") env = {"PATH": f"{bin_dir}:/usr/bin:/bin", "OUT": str(tmp_path), "GH_TOKEN": token, "DEFAULT_TOKEN": default_token, - "PR": "1", "REPO": "o/r", "GH_LOG": str(tmp_path / "gh.log")} + "PR": "1", "REPO": "o/r", "GH_LOG": str(tmp_path / "gh.log"), + **({"GH_WRITE_FAILS": "1"} if write_fails else {})} done = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True, check=False) return done, (tmp_path / "gh.log").read_text(encoding="utf-8") if (tmp_path / "gh.log").exists() else "" @@ -390,3 +392,105 @@ def test_sticky_comment_refuses_a_token_whose_login_is_unreadable(tmp_path): def test_sticky_comment_posts_with_the_default_token(tmp_path): done, log = _run_sticky(tmp_path, "default-token", "default-token") assert done.returncode == 0 and "POST" in log and "api user" not in log + + +def test_sticky_comment_that_cannot_be_posted_is_a_warning_not_a_failed_job(tmp_path): + done, log = _run_sticky(tmp_path, "default-token", "default-token", write_fails=True) + assert "POST" in log and done.returncode == 0 and "::warning title=dokimi report::" in done.stdout + + +def test_sticky_comment_body_stays_below_the_comment_limit(tmp_path): + done, _ = _run_sticky(tmp_path, "default-token", "default-token", body="| row |\n" * 20000) + comment = (tmp_path / "summary" / "comment.md").read_text(encoding="utf-8") + assert done.returncode == 0 and len(comment) < 65536 and "job summary" in comment + + +def test_a_run_stopped_short_is_not_a_pass(tmp_path): + _doc(tmp_path, "a.json", "catalog check", verdict="pass") + _doc(tmp_path, "b.json", "skill routing", status="budget_stopped", reason="spend cap reached") + out = summarize([str(tmp_path)]) + assert [c["state"] for c in out["checks"]] == ["pass", "budget_stopped"] + assert out["overall"] == "budget_stopped" + assert "::warning title=skill routing::budget_stopped: spend cap reached" in render(out, "annotations") + assert schema_errors("report_summary.v1", out) == [] + + +def test_a_failing_verdict_outranks_a_stopped_status(tmp_path): + _doc(tmp_path, "a.json", "catalog eval", status="partial", verdict="fail") + assert summarize([str(tmp_path)])["checks"][0]["state"] == "fail" + + +def test_an_unpriced_spend_is_not_shown_as_an_amount_or_added_to_the_total(tmp_path): + _doc(tmp_path, "a.json", "skill routing", cost={"spent_usd": 0.0, "cap_usd": 1.0, "priced": False}) + _doc(tmp_path, "b.json", "repo hits", cost={"spent_usd": 2.0, "cap_usd": 5.0, "priced": True}) + out = summarize([str(tmp_path)]) + md = render(out, "markdown") + assert out["spent_usd"] is None and "unpriced / 1.0" in md and "2.0 / 5.0" in md and "0.0 / 1.0" not in md + + +def test_skill_routing_json_leaves_no_remainder_for_an_unpriced_run(tmp_path): + d, c = _routed_skill(tmp_path) + cost = {**_COST, "usd": None} + with patch("dokimi.novelty.cost.client", return_value=object()), \ + patch("dokimi.novelty.client.roll_up_cost", return_value=cost), \ + patch("dokimi.novelty.routing.model_routing", return_value=_ARM): + _, out, _ = _main("skill", "routing", str(d), "--competitors", str(c), "--queries-from-headings", + "--model", "m", "--max-usd", "1", "--out", str(tmp_path / "r.json"), "--json") + doc = json.loads(out) + assert doc["cost"]["priced"] is False and doc["cost"]["remaining_usd"] is None + + +def test_markdown_lists_each_finding_with_its_location_and_escapes_html(tmp_path): + doc = cli_output.envelope("repo scan", verdict="fail", subjects=[{"subject": "AGENTS.md", "verdict": "fail", "findings": [ + cli_output.finding("error", "claim", "no
| here", "AGENTS.md", 7)]}]) + p = tmp_path / "c.json" + p.write_text(json.dumps(doc), encoding="utf-8") + md = render(summarize([str(p)]), "markdown") + assert "| AGENTS.md | repo scan | error | AGENTS.md:7 | claim | no <details> \\| here |" in md + + +def test_markdown_caps_the_listed_findings(tmp_path): + findings = [cli_output.finding("warn", "w", f"m{i}", "f.md", i + 1) for i in range(103)] + doc = cli_output.envelope("repo scan", subjects=[{"subject": "s", "verdict": None, "findings": findings}]) + p = tmp_path / "c.json" + p.write_text(json.dumps(doc), encoding="utf-8") + md = render(summarize([str(p)]), "markdown") + assert "| m99 |" in md and "| m100 |" not in md and "3 more findings" in md + + +def _git_catalog(tmp: Path) -> Path: + root = _catalog(tmp, broken=True) + git = ["git", "-c", "user.name=t", "-c", "user.email=t@t", "-C", str(root)] + subprocess.run([*git[:5], "init", "-q", str(root)], check=True) + subprocess.run([*git, "add", "."], check=True) + subprocess.run([*git, "commit", "-q", "-m", "base"], check=True) + (root / "skills" / "alpha" / "SKILL.md").write_text( + "---\nname: alpha\ndescription: Use for alpha tasks, changed.\n---\nRun the checks.\n", encoding="utf-8") + return root + + +def test_catalog_eval_json_leaves_a_skill_outside_the_selection_unjudged(tmp_path): + root = _git_catalog(tmp_path) + code, out, _ = _main("catalog", "eval", str(root), "--out-dir", str(tmp_path / "out"), "--changed-from", "HEAD", "--json") + doc = json.loads(out) + subjects = {s["subject"]: s for s in doc["subjects"]} + assert code == 0 and doc["verdict"] == "pass" + assert subjects["beta"]["verdict"] is None and subjects["beta"]["findings"] == [] + assert subjects["alpha"]["verdict"] == "pass" + annotations = render(summarize([_write(tmp_path, doc)]), "annotations") + assert "beta" not in annotations + + +def _write(tmp: Path, doc: dict) -> str: + p = tmp / "eval.json" + p.write_text(json.dumps(doc), encoding="utf-8") + return str(p) + + +def test_catalog_eval_json_findings_point_at_the_skill_file_under_the_given_root(tmp_path, monkeypatch): + _catalog(tmp_path, broken=True) + monkeypatch.chdir(tmp_path) + code, out, _ = _main("catalog", "eval", "catalog", "--out-dir", str(tmp_path / "out"), "--json") + doc = json.loads(out) + files = {f["file"] for s in doc["subjects"] for f in s["findings"]} + assert code == 1 and files == {"catalog/skills/beta/SKILL.md"} From 4eea5876091776635eada3a4987ce4b6fa86297f Mon Sep 17 00:00:00 2001 From: vchamarthi Date: Tue, 17 Nov 2026 18:05:51 +0000 Subject: [PATCH 5/6] fix(report): sticky lookup failure, task finding paths, cost.priced type - actions/report no longer posts a second comment when the lookup of the existing one fails (even after printing an id); it warns and skips. - catalog eval --json points a task finding at its task.toml, task.yaml or task.md instead of a nonexistent /SKILL.md. - report summary treats a cost.priced that is not a boolean or null as an invalid row, so the JSON stays valid against report_summary.v1. Signed-off-by: vchamarthi --- actions/report/action.yml | 9 +++- ...261116_120000_vchamarthi_report-summary.md | 6 +-- dokimi/dokimi/commands/catalog_eval.py | 8 ++-- dokimi/dokimi/report/summary.py | 3 +- dokimi/tests/test_report_summary.py | 45 +++++++++++++++++-- 5 files changed, 59 insertions(+), 12 deletions(-) diff --git a/actions/report/action.yml b/actions/report/action.yml index fb8850f50..9365ba446 100644 --- a/actions/report/action.yml +++ b/actions/report/action.yml @@ -75,8 +75,13 @@ runs: exit 1 } fi - id=$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ - --jq ".[] | select(.user.login == \"$me\" and (.body | startswith(\"$marker\"))) | .id" | head -n1) || id= + # A failed lookup is not proof that no comment exists: skip publishing instead of posting a duplicate. + ids=$(gh api --paginate "repos/$REPO/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"$me\" and (.body | startswith(\"$marker\"))) | .id") || { + echo "::warning title=dokimi report::the sticky comment lookup failed; the summary is in the job summary" + exit 0 + } + id=$(printf '%s\n' "$ids" | head -n1) # A token that cannot write (a pull request from a fork) must not turn a green job red. if [ -n "$id" ]; then gh api -X PATCH "repos/$REPO/issues/comments/$id" -F "body=@$OUT/summary/comment.md" >/dev/null || diff --git a/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md b/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md index 940747a7f..b583b4f19 100644 --- a/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md +++ b/dokimi/changelog.d/20261116_120000_vchamarthi_report-summary.md @@ -1,6 +1,6 @@ ### Added -- **`dokimi report summary` merges `cli_output.v1` documents into one summary** (#749, the renderer). It takes files or directories of `*.json` and writes `--format markdown` (a check table and a subject table, usable as a job summary, a PR comment or terminal output), `json` (`report_summary.v1`, with a bundled schema) or `annotations` (`::error`/`::warning` lines). It reads envelope keys only, so a new leaf needs no change in it. A missing, unreadable, malformed or non-`cli_output.v1` input, and a directory with no `*.json` file, stays a row (`not_run` or `invalid`) and makes the overall state bad; it is never dropped or shown as a pass. A document whose status is `budget_stopped`, `partial` or `cancelled` is that state, not a pass, and gets a `::warning`; an unpriced spend shows as `unpriced` and leaves the total empty; the Markdown lists each finding (file:line, code, message, the first 100). It computes no verdict and exits 0 once written: the gate stays each leaf's own exit code. Stdlib only, and it reads no `GITHUB_*` variable. -- **`catalog check`, `catalog eval` and `skill routing` take `--json`** (#749, #94 F2): one `cli_output.v1` document on stdout, with one subject per skill (and per task for `catalog check`). Exit codes are unchanged. `catalog eval --json` leaves a skill outside `--changed-from` unjudged (`verdict: null`, no findings) and points a finding at its `SKILL.md` under the given root. `skill routing --json` is report only (`verdict: null`) and carries `queries_self_derived` next to its scores; its human summary goes to stderr. `skill routing --json` still prints its document when `--max-usd` or a model error stops the run (status `budget_stopped` or `partial`) and then exits non-zero as before; its subject is the loaded skill's name. `catalog check --json` refuses `--format json` and `--jsonl`. +- **`dokimi report summary` merges `cli_output.v1` documents into one summary** (#749, the renderer). It takes files or directories of `*.json` and writes `--format markdown` (a check table and a subject table, usable as a job summary, a PR comment or terminal output), `json` (`report_summary.v1`, with a bundled schema) or `annotations` (`::error`/`::warning` lines). It reads envelope keys only, so a new leaf needs no change in it. A missing, unreadable, malformed (including a `cost.priced` that is not a boolean) or non-`cli_output.v1` input, and a directory with no `*.json` file, stays a row (`not_run` or `invalid`) and makes the overall state bad; it is never dropped or shown as a pass. A document whose status is `budget_stopped`, `partial` or `cancelled` is that state, not a pass, and gets a `::warning`; an unpriced spend shows as `unpriced` and leaves the total empty; the Markdown lists each finding (file:line, code, message, the first 100). It computes no verdict and exits 0 once written: the gate stays each leaf's own exit code. Stdlib only, and it reads no `GITHUB_*` variable. +- **`catalog check`, `catalog eval` and `skill routing` take `--json`** (#749, #94 F2): one `cli_output.v1` document on stdout, with one subject per skill (and per task for `catalog check`). Exit codes are unchanged. `catalog eval --json` leaves a skill outside `--changed-from` unjudged (`verdict: null`, no findings) and points a finding at its `SKILL.md` (a task finding at its `task.toml`, `task.yaml` or `task.md`) under the given root. `skill routing --json` is report only (`verdict: null`) and carries `queries_self_derived` next to its scores; its human summary goes to stderr. `skill routing --json` still prints its document when `--max-usd` or a model error stops the run (status `budget_stopped` or `partial`) and then exits non-zero as before; its subject is the loaded skill's name. `catalog check --json` refuses `--format json` and `--jsonl`. - **A `cli_output.v1` subject may carry `findings`** (`level`, `code`, `message`, `file`, `line`). `catalog check`, `catalog eval`, `repo validate` and `repo scan` fill them; `repo scan --json` puts each failing claim at its file and line and adds the claim counts to `summary`. `report summary --format annotations` gives each finding a `file=`/`line=` annotation, and the Markdown shows the scalar `summary` values of each document. -- **`actions/report`** places the summary: it runs `dokimi report summary` over `out-dir`, appends the Markdown to the job summary, prints the annotations, keeps one sticky PR comment with `comment: sticky`, cut below the comment limit and only a warning when it cannot be posted, and uploads `out-dir`. No parsing and no verdict in YAML. `repo-eval.yml` renders `repo validate` and `repo scan` with the same renderer instead of two inline Python blocks. +- **`actions/report`** places the summary: it runs `dokimi report summary` over `out-dir`, appends the Markdown to the job summary, prints the annotations, keeps one sticky PR comment with `comment: sticky`, cut below the comment limit, only a warning when it cannot be posted, and no comment at all when the lookup of the existing one fails, and uploads `out-dir`. No parsing and no verdict in YAML. `repo-eval.yml` renders `repo validate` and `repo scan` with the same renderer instead of two inline Python blocks. diff --git a/dokimi/dokimi/commands/catalog_eval.py b/dokimi/dokimi/commands/catalog_eval.py index 36a4eaf46..572f240ec 100644 --- a/dokimi/dokimi/commands/catalog_eval.py +++ b/dokimi/dokimi/commands/catalog_eval.py @@ -208,8 +208,9 @@ def _emit(report: dict[str, Any], out: Path, root: Path) -> None: ``data`` holds the per-skill evaluation state (``no_tasks``, ``not_measured``, ``checks_only``) and the findings that belong to no skill; the full report stays in the receipts under ``--out-dir``. A finding's file is - under ``root`` as given, and a skill directory becomes its ``SKILL.md``. With ``--changed-from``, a skill outside - the selection was not evaluated: its verdict is null and it is left out of ``reason``. + under ``root`` as given, and a skill or task directory becomes its ``SKILL.md`` or task definition. With + ``--changed-from``, a skill outside the selection was not evaluated: its verdict is null and it is left out of + ``reason``. """ from dokimi import cli_output from dokimi.external.catalog_eval import shared_problems @@ -217,7 +218,8 @@ def _emit(report: dict[str, Any], out: Path, root: Path) -> None: def finding(item: dict[str, Any]) -> dict[str, Any]: path = root / item["path"] if item.get("path") else None if path is not None and path.is_dir(): - path = path / "SKILL.md" + path = next((path / name for name in ("SKILL.md", "task.toml", "task.yaml", "task.md") + if (path / name).is_file()), path) return cli_output.finding(item["level"], item["code"], item["message"], path and path.as_posix()) selected = set(report["selection"]["selected"]) diff --git a/dokimi/dokimi/report/summary.py b/dokimi/dokimi/report/summary.py index f5f70cd9f..16cfd332a 100644 --- a/dokimi/dokimi/report/summary.py +++ b/dokimi/dokimi/report/summary.py @@ -82,7 +82,8 @@ def _malformed(doc: dict[str, Any]) -> str | None: cost = doc.get("cost") if cost is not None and (not isinstance(cost, dict) or any( v is not None and (isinstance(v, bool) or not isinstance(v, (int, float))) - for v in (cost.get("spent_usd"), cost.get("cap_usd")))): + for v in (cost.get("spent_usd"), cost.get("cap_usd"))) or not ( + cost.get("priced") is None or isinstance(cost["priced"], bool))): return "cost" return None diff --git a/dokimi/tests/test_report_summary.py b/dokimi/tests/test_report_summary.py index e34ecb7ba..eb5ac5e9b 100644 --- a/dokimi/tests/test_report_summary.py +++ b/dokimi/tests/test_report_summary.py @@ -239,6 +239,13 @@ def test_malformed_envelope_fields_are_invalid_rows_not_a_crash(tmp_path): assert (rows[field]["state"], rows[field]["reason"]) == ("invalid", f"{field} malformed") +def test_a_non_boolean_priced_is_an_invalid_row(tmp_path): + _doc(tmp_path, "a.json", "repo hits", verdict="pass", cost={"spent_usd": 0.0, "priced": "false"}) + _doc(tmp_path, "b.json", "repo scan", verdict="pass", cost={"spent_usd": 0.0, "priced": None}) + out = summarize([str(tmp_path)]) + assert [(c["state"], c["reason"]) for c in out["checks"]] == [("invalid", "cost malformed"), ("pass", None)] + + def test_two_documents_of_one_kind_keep_each_subject_verdict(tmp_path): _doc(tmp_path, "a.json", "memory lint", subjects=[{"subject": "s1", "verdict": "fail"}]) _doc(tmp_path, "b.json", "memory lint", subjects=[{"subject": "s1", "verdict": "pass"}]) @@ -363,7 +370,7 @@ def test_the_report_action_uploads_after_a_failed_comment_and_finds_its_own_comm assert "gh api user" in sticky and '.user.login == \\"$me\\"' in sticky -def _run_sticky(tmp_path, token, default_token, body="x\n", write_fails=False): +def _run_sticky(tmp_path, token, default_token, body="x\n", write_fails=False, lookup=""): """The sticky step's script against a fake ``gh`` that has no ``/user`` and logs its calls.""" import yaml @@ -373,14 +380,15 @@ def _run_sticky(tmp_path, token, default_token, body="x\n", write_fails=False): bin_dir.mkdir() gh = bin_dir / "gh" gh.write_text('#!/bin/sh\necho "$@" >> "$GH_LOG"\n[ "$2" = user ] && exit 1\n' + '[ "$2" = --paginate ] && [ -n "$GH_LOOKUP" ] && { echo 7; [ "$GH_LOOKUP" = fail ] && exit 1; exit 0; }\n' '[ -n "$GH_WRITE_FAILS" ] && [ "$2" = -X ] && exit 1\nexit 0\n', encoding="utf-8") gh.chmod(0o755) (tmp_path / "summary").mkdir() (tmp_path / "summary" / "summary.md").write_text(body, encoding="utf-8") env = {"PATH": f"{bin_dir}:/usr/bin:/bin", "OUT": str(tmp_path), "GH_TOKEN": token, "DEFAULT_TOKEN": default_token, "PR": "1", "REPO": "o/r", "GH_LOG": str(tmp_path / "gh.log"), - **({"GH_WRITE_FAILS": "1"} if write_fails else {})} - done = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True, check=False) + **({"GH_WRITE_FAILS": "1"} if write_fails else {}), **({"GH_LOOKUP": lookup} if lookup else {})} + done = subprocess.run(["bash", "-o", "pipefail", "-c", script], env=env, capture_output=True, text=True, check=False) return done, (tmp_path / "gh.log").read_text(encoding="utf-8") if (tmp_path / "gh.log").exists() else "" @@ -399,6 +407,17 @@ def test_sticky_comment_that_cannot_be_posted_is_a_warning_not_a_failed_job(tmp_ assert "POST" in log and done.returncode == 0 and "::warning title=dokimi report::" in done.stdout +def test_sticky_comment_lookup_that_fails_after_an_id_posts_nothing(tmp_path): + done, log = _run_sticky(tmp_path, "default-token", "default-token", lookup="fail") + assert done.returncode == 0 and "POST" not in log and "PATCH" not in log + assert "::warning title=dokimi report::the sticky comment lookup failed" in done.stdout + + +def test_sticky_comment_updates_the_comment_the_lookup_found(tmp_path): + done, log = _run_sticky(tmp_path, "default-token", "default-token", lookup="ok") + assert done.returncode == 0 and "PATCH repos/o/r/issues/comments/7" in log and "POST" not in log + + def test_sticky_comment_body_stays_below_the_comment_limit(tmp_path): done, _ = _run_sticky(tmp_path, "default-token", "default-token", body="| row |\n" * 20000) comment = (tmp_path / "summary" / "comment.md").read_text(encoding="utf-8") @@ -494,3 +513,23 @@ def test_catalog_eval_json_findings_point_at_the_skill_file_under_the_given_root doc = json.loads(out) files = {f["file"] for s in doc["subjects"] for f in s["findings"]} assert code == 1 and files == {"catalog/skills/beta/SKILL.md"} + + +def test_catalog_eval_json_task_findings_point_at_the_task_definition(tmp_path): + from dokimi.commands.catalog_eval import _emit + + (tmp_path / "skills" / "alpha").mkdir(parents=True) + (tmp_path / "skills" / "alpha" / "SKILL.md").write_text("x", encoding="utf-8") + (tmp_path / "tasks" / "t1").mkdir(parents=True) + (tmp_path / "tasks" / "t1" / "task.toml").write_text("x", encoding="utf-8") + row = {"name": "alpha", "evaluation": "no_tasks", "checks": {"status": "pass", "findings": [ + {"level": "warn", "code": "undeclared-task", "message": "m", "path": "tasks/t1"}, + {"level": "warn", "code": "other", "message": "m", "path": "skills/alpha"}]}} + report = {"status": "pass", "tier": "static", "problems": [], "skills": [row], + "selection": {"selected": ["alpha"]}} + out = tmp_path / "out" + out.mkdir() + with patch("dokimi.cli_output.emit_leaf") as leaf: + _emit(report, out, tmp_path) + files = [f["file"] for f in leaf.call_args.kwargs["subjects"][0]["findings"]] + assert files == [(tmp_path / "tasks/t1/task.toml").as_posix(), (tmp_path / "skills/alpha/SKILL.md").as_posix()] From fea499cd563156f3886d5d14462ba34a58a9a543 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 17 Nov 2026 19:15:13 +0000 Subject: [PATCH 6/6] Show repo scan warnings as findings in the report summary Signed-off-by: Anton Volkov --- dokimi/dokimi/commands/repo_scan.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dokimi/dokimi/commands/repo_scan.py b/dokimi/dokimi/commands/repo_scan.py index c180cf1c3..dd7d7eb73 100644 --- a/dokimi/dokimi/commands/repo_scan.py +++ b/dokimi/dokimi/commands/repo_scan.py @@ -168,5 +168,8 @@ def scan_subjects(payload: dict) -> list[dict]: message = f"{claim.get('kind')}: `{str(claim.get('claim'))[:80]}`: {claim.get('why')}" by_file.setdefault(file, []).append(finding("warn", str(claim.get("status")), message, file, line)) files = [str(f.get("path", f)) if isinstance(f, dict) else str(f) for f in payload.get("files") or []] - return [{"subject": f, "verdict": None, "findings": by_file.get(f, [])} - for f in sorted(set(files) | set(by_file))] + names = sorted(set(files) | set(by_file)) + if names: + by_file.setdefault(names[0], []).extend( + finding("warn", "scan-warning", str(w)) for w in payload.get("warnings") or []) + return [{"subject": f, "verdict": None, "findings": by_file.get(f, [])} for f in names]