Repository navigation
feat(catalog): catalog provenance matches skill copies to an upstream version #838
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
cdf421f
b434f31
185fee6
21b99cc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,8 @@ | ||
| <!-- | ||
| Copyright (C) 2026 Intel Corporation | ||
| SPDX-License-Identifier: Apache-2.0 | ||
| --> | ||
|
|
||
| ### Added | ||
|
|
||
| - **`catalog provenance` matches each skill copy to the upstream version it came from** — offline, from the `--upstream` git checkouts or `--upstream-manifest` files you name (dokimi ships no upstream list). A copy is matched by its git blob sha against the upstream history, then by frontmatter name and word Jaccard similarity ≥ 0.8. The report gives the origin skill and version, how many commits and months behind upstream head, and whether the copy was edited. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| # Copyright (C) 2026 Intel Corporation | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| """``catalog provenance``: which upstream version each skill copy under ``--root`` came from, and how far behind it is. | ||
|
|
||
| Read-only and offline: the upstreams are the caller's (``--upstream`` checkouts, ``--upstream-manifest`` files). The | ||
| matching is in :mod:`dokimi.external.upstream`; this module reads the catalog and prints the rows. | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
| import json | ||
| from pathlib import Path | ||
|
|
||
| from dokimi import cli_flags | ||
| from dokimi.utils import InputError | ||
|
|
||
|
|
||
| def _rel(skill_dir: Path, root: Path) -> str: | ||
| try: | ||
| return skill_dir.relative_to(root).as_posix() | ||
| except ValueError: | ||
| return skill_dir.as_posix() | ||
|
|
||
|
|
||
| def _behind_text(row: dict) -> str: | ||
| if row["match"] is None: | ||
| return "-" | ||
| if row["at_head"]: | ||
| return "at head" | ||
| commits = "" if row["behind_commits"] is None else f"{row['behind_commits']} commits" | ||
| months = "" if row["behind_months"] is None else f"{row['behind_months']} mo" | ||
| return ", ".join(part for part in (commits, months) if part) or "unknown" | ||
|
|
||
|
|
||
| def _text(report: dict) -> str: | ||
| summary = report["summary"] | ||
| lines = [] | ||
| for up in report["upstreams"]: | ||
| head = f"head {up['head_commit']} {up['head_date'] or ''}".strip() if up["head_commit"] else "no head" | ||
| note = " (shallow clone: older versions are not seen)" if up["history"] == "shallow" else "" | ||
| lines.append(f"# upstream {up['name']}: {head}, history {up['history']}{note}") | ||
| median = summary["median_months_behind"] | ||
| lines.append( | ||
| f"# skills ({summary['skills']} under {report['root']}: {summary['exact']} exact, " | ||
| f"{summary['similar']} similar, {summary['unmatched']} unmatched; {summary['edited']} edited; " | ||
| f"{summary['at_head']} at head; median {'-' if median is None else median} months behind)" | ||
| ) | ||
| lines.append("skill\tmatch\tupstream\tversion\tbehind\tedited") | ||
| for row in report["skills"]: | ||
| upstream_text = "-" if row["upstream"] is None else f"{row['upstream']}:{row['path']}" | ||
| version = "-" if row["commit"] is None else f"{row['commit']}{' ' + row['date'] if row['date'] else ''}" | ||
| if row["version"]: | ||
|
vchamarthi marked this conversation as resolved.
Resolved
|
||
| version = f"v{row['version']} {version}" | ||
| edited = "-" if row["edited"] is None else "yes" if row["edited"] else "no" | ||
| lines.append( | ||
| f"{row['skill']}\t{row['match'] or 'unmatched'}\t{upstream_text}\t{version}\t" | ||
| f"{_behind_text(row)}\t{edited}" | ||
| ) | ||
| return "\n".join(lines) | ||
|
|
||
|
|
||
| def cmd_catalog_provenance(args: argparse.Namespace) -> None: | ||
| from dokimi.external import read_skill_manifest | ||
| from dokimi.external import upstream | ||
|
|
||
| root = Path(args.root) | ||
| if not root.is_dir(): | ||
| raise InputError(f"--root is not a directory: {root}") | ||
| if not args.upstream and not args.upstream_manifest: | ||
| raise InputError("catalog provenance needs --upstream CHECKOUT (a git checkout) or " | ||
| "--upstream-manifest FILE: dokimi ships no upstream list") | ||
| index = upstream.build(args.upstream or (), args.upstream_manifest or ()) | ||
| try: | ||
| entries = read_skill_manifest(root) | ||
| except ValueError as exc: | ||
| raise InputError(f"skills.yaml: {exc}") from exc | ||
| rows = [] | ||
| for entry in entries: | ||
| if not entry.loaded or entry.skill is None: | ||
| continue # a skill that does not load has no bytes to match: `catalog check` reports it | ||
|
Comment on lines
+81
to
+82
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. A copy that does not load leaves no trace in the report. A Suggestion: match these by their bytes too, or at least count them in |
||
| skill = entry.skill | ||
| match = index.match(skill.path.read_bytes(), skill.name) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. A copy with no frontmatter
Suggestion: pass the copy's frontmatter |
||
| rows.append({"skill": _rel(skill.path.parent, root), "name": skill.name, **upstream.to_row(match)}) | ||
| report = { | ||
| "root": str(root), | ||
| "upstreams": upstream.upstream_rows(index), | ||
| "skills": rows, | ||
| "summary": upstream.summary(rows), | ||
| } | ||
| if args.format == "json": | ||
| print(json.dumps(report, indent=2)) | ||
| else: | ||
| print(_text(report)) | ||
|
|
||
|
|
||
| def register(catalog_sub) -> None: | ||
| """Add ``catalog provenance`` to the ``catalog`` group.""" | ||
| prov_p = catalog_sub.add_parser( | ||
| "provenance", | ||
| help="For each skill under --root: the upstream version it copies, how far behind head, and whether it was edited", | ||
| ) | ||
| prov_p.add_argument("--root", required=True, help="Local checkout of the skills to match") | ||
| cli_flags.add( | ||
| prov_p, "upstream", "--upstream", action="append", default=None, metavar="CHECKOUT", | ||
| help="A local git checkout of an upstream (its history gives the versions). Repeatable; the first one " | ||
| "wins a tie. Nothing is fetched: clone it yourself, full history for an exact count", | ||
| ) | ||
| cli_flags.add( | ||
| prov_p, "upstream", "--upstream-manifest", action="append", default=None, metavar="FILE", | ||
| help="A JSON manifest of one upstream: {\"schema\": \"dokimi.upstream.v1\", \"name\", \"head\": {commit, date}, " | ||
| "\"skills\": [{path, content, head?, commit?, date?}]}. Repeatable", | ||
| ) | ||
| prov_p.add_argument( | ||
| "--format", choices=["text", "json"], default="text", | ||
| help="text: one line per skill (default); json: the upstreams, one row per skill and the summary", | ||
| ) | ||
| prov_p.set_defaults(func=cmd_catalog_provenance) | ||
Uh oh!
There was an error while loading. Please reload this page.