diff --git a/.agents/rules/aitools-verification.md b/.agents/rules/aitools-verification.md new file mode 100644 index 00000000000..f614015bf6f --- /dev/null +++ b/.agents/rules/aitools-verification.md @@ -0,0 +1,17 @@ +--- +description: Fast and full verification commands for the aitools command group +globs: + - "cmd/aitools/**" + - "libs/aitools/**" + - "experimental/aitools/**" +paths: + - "cmd/aitools/**" + - "libs/aitools/**" + - "experimental/aitools/**" +--- + +# aitools verification + +**RULE: After every edit under `cmd/aitools`, `libs/aitools` or `experimental/aitools`, run `./task aitools-quick`.** It runs golangci-lint and the unit tests of the aitools packages and compiles their importers, in parallel, and prints a summary for every step and detailed output for failed steps. `-- --json ` also writes a machine-readable report, with file:line findings when available. + +**RULE: Before opening a PR that touches aitools, run `./task aitools-full`.** It adds the importers' unit tests, the acceptance tests that exercise aitools and the deadcode check, which catch output and wiring changes the quick tier misses. diff --git a/.cursor/rules/aitools-verification.mdc b/.cursor/rules/aitools-verification.mdc new file mode 120000 index 00000000000..39d8195b4ef --- /dev/null +++ b/.cursor/rules/aitools-verification.mdc @@ -0,0 +1 @@ +../../.agents/rules/aitools-verification.md \ No newline at end of file diff --git a/Taskfile.yml b/Taskfile.yml index 3d61ad991b9..875944ab0c7 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -92,6 +92,22 @@ tasks: - task: lint-q-go-root - task: lint-python + aitools-quick: + desc: Fast aitools check to run after every edit (lint and unit tests of aitools, build of its importers) + # No sources: Go's build and test caches decide what re-runs. + cmds: + - "uv run --script tools/aitools_quick.py {{.CLI_ARGS}}" + + aitools-full: + desc: Full aitools tier to run before a PR (aitools-quick plus importer tests, aitools acceptance tests and deadcode) + cmds: + - "uv run --script tools/aitools_quick.py --full {{.CLI_ARGS}}" + + aitools-quick-bench: + desc: Re-measure aitools-quick wall time after simulated edits (and, with `detect`, its detection rate) + cmds: + - "uv run --script tools/aitools_quick_bench.py {{.CLI_ARGS}}" + lint-python: desc: Lint first-party Python scripts with ruff sources: diff --git a/tools/aitools_quick.py b/tools/aitools_quick.py new file mode 100644 index 00000000000..8d6f65b0486 --- /dev/null +++ b/tools/aitools_quick.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.12" +# /// +""" +Fast check for the aitools command group, meant to run after every edit. + +Runs golangci-lint (which includes go vet's analyzers) and the unit tests of the +aitools packages, and compiles the packages that import them, in parallel. It +prints a summary for every step and detailed output for failed steps; file:line +findings are captured when available. Packages changed on the branch (diff +against the merge base with origin/main, or main when there is no origin/main, +as tools/lintdiff.py computes it) are linted and tested too, so an edit to a shared package such as libs/telemetry runs that +package's tests. Beyond that there is no package selection: Go's build and test +caches re-run exactly the packages whose transitive inputs changed. + +The unit tests run in a sanitized test environment: the host's coding-agent +variables and DATABRICKS_* variables are removed and HOME points at a dedicated +directory. This prevents agent-detection variables from changing aitools +behavior relative to CI. The acceptance tests run with the host environment: +the harness gives each test its own HOME and unsets the agent variables itself. + +With --full (`./task aitools-full`) it also runs the importers' unit tests, the +acceptance tests that exercise aitools and the deadcode check: the slower tier to +run before a PR. + +Usage: + ./task aitools-quick + ./task aitools-quick -- --json report.json + ./task aitools-full +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor + +from lintdiff import changed_go_dirs, go_dirs + +PACKAGES = ["./cmd/aitools/...", "./libs/aitools/...", "./experimental/aitools/..."] + +# Packages outside aitools that import it. cmd/root calls aitools from the root +# command's pre-run, so a change there can break every command. The fake agent +# CLIs of the acceptance tests write agent config through aitools. +IMPORTERS = [ + ".", + "./acceptance/internal/fakeagent", + "./cmd", + "./cmd/air", + "./cmd/apps", + "./cmd/experimental", + "./cmd/root", + "./cmd/version", +] + +# Acceptance tests that run aitools: its own directories, plus the goldens that +# print aitools help or tips. The apps tests cover the `apps init` skills tip. +ACCEPTANCE = [ + "TestAccept/apps", + "TestAccept/cmd/aitools", + "TestAccept/experimental/aitools", + "TestAccept/help$", + "TestAccept/cmd/version", + "TestAccept/quickstart", + # All of air: besides its help goldens, image-push records the aitools install hint. + "TestAccept/air", +] + +# The agent variables aitools reads beyond the detection list in +# acceptance/acceptance_test.go: Claude Code's child-session marker, which gates +# the native hint, and the agent config-dir overrides. +AGENT_EXTRA_ENV = { + "CLAUDE_CODE_CHILD_SESSION", + "CLAUDE_CODE_SESSION_ID", + "CLAUDE_CONFIG_DIR", + "CODEX_HOME", + "COPILOT_HOME", + "GEMINI_CLI_HOME", + "GOOSE_PATH_ROOT", + "PI_CODING_AGENT_DIR", + "XDG_CONFIG_HOME", +} + +MODULE = "github.com/databricks/cli" + +# file.go:line[:col]: message, as printed by the compiler, go vet and golangci-lint. +FINDING = re.compile(r"^(?P[^\s:][^:]*\.go):(?P\d+)(?::(?P\d+))?: (?P.+)$") + + +def run(name, cmd, env=None): + start = time.monotonic() + proc = subprocess.run(cmd, capture_output=True, text=True, env=env) + return { + "step": name, + "cmd": " ".join(cmd), + "ok": proc.returncode == 0, + "seconds": round(time.monotonic() - start, 2), + "output": proc.stdout + proc.stderr, + } + + +def agent_env(src): + """Return the agent variables to remove: the ones the acceptance harness + unsets (read from its source, src, so the two lists can't drift) plus + AGENT_EXTRA_ENV. + + >>> src = '// Unset AI-agent detection env vars\\nfor _, v := range []string{\\n\\t"AGENT",\\n\\t"KIRO",\\n} {' + >>> sorted(agent_env(src) - AGENT_EXTRA_ENV) + ['AGENT', 'KIRO'] + """ + m = re.search(r"Unset AI-agent detection env vars.*?range \[\]string\{(?P.*?)\}", src, re.DOTALL) + if not m: + raise SystemExit("aitools-quick: cannot find the agent variable list in acceptance/acceptance_test.go") + return set(re.findall(r'"(\w+)"', m["list"])) | AGENT_EXTRA_ENV + + +def findings(output, pkg_dir=""): + """Return file:line findings from compiler, vet, lint or test output. + + Test logs name only the file, so a bare file name is joined to pkg_dir. + + >>> findings("# pkg\\ncmd/aitools/x.go:3:5: undefined: y\\nok") + [{'file': 'cmd/aitools/x.go', 'line': 3, 'col': 5, 'message': 'undefined: y'}] + >>> findings(" hint_test.go:375: want x", "libs/aitools/agents") + [{'file': 'libs/aitools/agents/hint_test.go', 'line': 375, 'col': None, 'message': 'want x'}] + """ + result = [] + for line in output.splitlines(): + m = FINDING.match(line.strip()) + if m: + col = int(m["col"]) if m["col"] else None + file = m["file"] if "/" in m["file"] else os.path.join(pkg_dir, m["file"]).replace(os.sep, "/") + result.append({"file": file, "line": int(m["line"]), "col": col, "message": m["msg"]}) + return result + + +def package_dir(pkg): + """Return the repository directory of an import path. + + >>> package_dir("github.com/databricks/cli/libs/aitools/agents") + 'libs/aitools/agents' + >>> package_dir("github.com/databricks/cli") + '' + """ + return pkg.removeprefix(MODULE).removeprefix("/") + + +def test_failures(output): + """Return failing tests and their output from `go test -json` output. + + Build failures are reported as a pseudo-test named "(build)". A package that + fails with no failing test (a timeout, a panic in init, a TestMain exit) is + reported as a pseudo-test named "(package)" with the package's own output + and that of its tests that never finished, where a timeout's panic lands. + + >>> out = '''{"Action":"output","Package":"p","Test":"TestOK","Output":"ok\\\\n"} + ... {"Action":"pass","Package":"p","Test":"TestOK"} + ... {"Action":"output","Package":"p","Test":"TestSlow","Output":"panic: test timed out\\\\n"} + ... {"Action":"output","Package":"p","Output":"FAIL p\\\\n"} + ... {"Action":"fail","Package":"p"} + ... {"Action":"output","Package":"q","Test":"TestA","Output":"bad\\\\n"} + ... {"Action":"fail","Package":"q","Test":"TestA"} + ... {"Action":"fail","Package":"q"}''' + >>> test_failures(out) + [{'package': 'p', 'test': '(package)', 'output': 'panic: test timed out\\nFAIL p\\n'}, {'package': 'q', 'test': 'TestA', 'output': 'bad\\n'}] + """ + logs = {} + failed = [] + finished = set() + for line in output.splitlines(): + try: + ev = json.loads(line) + except json.JSONDecodeError: + continue + action = ev.get("Action") + if action in ("build-output", "build-fail"): + key = (ev.get("ImportPath", ""), "(build)") + else: + key = (ev.get("Package", ""), ev.get("Test") or "(package)") + if action in ("output", "build-output"): + logs.setdefault(key, []).append(ev.get("Output", "")) + elif action == "build-fail" or (action == "fail" and not ev.get("FailedBuild")): + failed.append(key) + if action in ("pass", "fail", "skip"): + finished.add(key) + # A failing subtest also fails its parent, and a failing test also fails its + # package; report only the innermost test. + names = {(p, t) for p, t in failed} + leaves = [ + (p, t) + for p, t in failed + if not any(q == p and (u.startswith(t + "/") or (t == "(package)" and u != t)) for q, u in names) + ] + + def output(p, t): + if t != "(package)": + return "".join(logs.get((p, t), [])) + keys = [k for k in logs if k[0] == p and k[1] != t and k not in finished] + [(p, t)] + return "".join(line for k in keys for line in logs.get(k, [])) + + return [{"package": p, "test": t, "output": output(p, t)} for p, t in leaves] + + +def select_packages(dirs): + """Return the ./dir package patterns among dirs outside PACKAGES. + + The acceptance package runs the whole acceptance suite and integration tests + need a workspace, so both are left to aitools-full; tools/ is its own module. + + >>> select_packages(["./libs/telemetry", "./cmd/aitools", "./acceptance", "./acceptance/internal", + ... "./integration/cmd", "./tools/testmask"]) + ['./libs/telemetry'] + """ + return sorted( + d + for d in dirs + if d.removeprefix("./").split("/")[0] not in ("acceptance", "integration", "tools") + and not any((d + "/").startswith(p.removesuffix("...")) for p in PACKAGES) + ) + + +def default_ref(): + """origin/main when it exists: a local main is not moved by `git fetch origin main`.""" + found = subprocess.run(["git", "rev-parse", "--verify", "--quiet", "origin/main"], capture_output=True) + return "origin/main" if found.returncode == 0 else "main" + + +def changed_packages(ref): + """Packages with Go files changed since the merge base with ref, committed or not.""" + dirs = changed_go_dirs(ref) + if dirs is None: + print(f"aitools-quick: cannot diff against {ref}; checking the aitools packages only", file=sys.stderr) + return [] + untracked = subprocess.run(["git", "ls-files", "--others", "--exclude-standard"], capture_output=True, text=True) + return select_packages(set(dirs) | set(go_dirs(untracked.stdout.split(), True))) + + +def user_cache_dir(): + """The per-user cache directory, as Go's os.UserCacheDir picks it.""" + if sys.platform == "win32": + return os.environ["LocalAppData"] + if sys.platform == "darwin": + return os.path.expanduser("~/Library/Caches") + return os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache") + + +def hermetic_env(home): + keys = ["GOCACHE", "GOENV", "GOMODCACHE", "GOPATH", "GOTOOLCHAIN"] + out = subprocess.run(["go", "env", "-json", *keys], capture_output=True, text=True, check=True) + pinned = json.loads(out.stdout) + with open("acceptance/acceptance_test.go") as f: + agent = agent_env(f.read()) + env = {k: v for k, v in os.environ.items() if k not in agent and not k.startswith("DATABRICKS_")} + # Pin the Go caches and the user's go env file (GOPROXY, GOFLAGS and the like + # set with `go env -w`) before moving HOME: otherwise every run would start + # cold and the tests would run under a different Go configuration than lint + # and build. + env.update({k: v for k, v in pinned.items() if v}) + env["HOME"] = home + env["USERPROFILE"] = home + return env + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--json", metavar="PATH", help="also write a machine-readable report to PATH") + parser.add_argument( + "--ref", help="ref whose merge base defines the changed packages (default: origin/main, or main without it)" + ) + parser.add_argument( + "--full", action="store_true", help="also run importer tests, aitools acceptance tests and deadcode" + ) + args = parser.parse_args() + name = "aitools-full" if args.full else "aitools-quick" + + # A fixed HOME that is not wiped between runs keeps the test cache valid: + # the cache key covers the env vars and files the tests read, HOME included. + # It lives in the user's cache directory, not the shared temp directory, + # where another account could create it first. + home = os.path.join(user_cache_dir(), "databricks-cli", "aitools-quick-home") + os.makedirs(home, mode=0o700, exist_ok=True) + + packages = PACKAGES + changed_packages(args.ref or default_ref()) + lint = [ + "go", + "tool", + "-modfile=tools/go.mod", + "golangci-lint", + "run", + "--allow-parallel-runners", + "-j=4", + *packages, + ] + if args.full: + # The same timeout as the Taskfile's acceptance tasks. + timeout = "-timeout=" + os.environ.get("LOCAL_TIMEOUT", "60m") + steps = [ + ("lint", lint, None), + ("test", ["go", "test", "-json", timeout, *packages, *IMPORTERS], hermetic_env(home)), + # The harness isolates each test itself, and it locates uv's cache + # and managed Pythons from the real HOME. + ("acceptance", ["go", "test", "-json", timeout, "./acceptance", "-run", "|".join(ACCEPTANCE)], None), + ("deadcode", ["uv", "run", "--script", "tools/check_deadcode.py"], None), + ] + else: + # The quick tier only checks that the importers still compile; their + # tests run in aitools-full. + steps = [ + ("lint", lint, None), + # A hung test fails in 2 minutes instead of go test's default 10. + ("test", ["go", "test", "-json", "-timeout=2m", *packages], hermetic_env(home)), + ("build", ["go", "build", *IMPORTERS], None), + ] + start = time.monotonic() + with ThreadPoolExecutor(len(steps)) as pool: + results = list(pool.map(lambda s: run(*s), steps)) + total = round(time.monotonic() - start, 2) + + for r in results: + if r["step"] in ("test", "acceptance"): + r["failures"] = test_failures(r["output"]) + # A failing build is reported through the compiler's own file:line output. + r["findings"] = [x for f in r["failures"] for x in findings(f["output"], package_dir(f["package"]))] + else: + r["findings"] = findings(r["output"]) + + for r in results: + print(f"{'ok ' if r['ok'] else 'FAIL'} {r['step']:<10} {r['seconds']:6.2f}s {r['cmd']}") + for r in results: + if r["ok"]: + continue + print(f"\n=== {r['step']} failed") + if r.get("failures"): + for f in r["failures"]: + print(f"--- {f['package']} {f['test']}\n{f['output'].rstrip()}") + else: + print(r["output"].rstrip()) + ok = all(r["ok"] for r in results) + print(f"\n{name}: {'PASS' if ok else 'FAIL'} in {total}s") + + if args.json: + report = {"ok": ok, "seconds": total, "steps": [{k: v for k, v in r.items() if k != "output"} for r in results]} + with open(args.json, "w") as f: + json.dump(report, f, indent=2) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/aitools_quick_bench.py b/tools/aitools_quick_bench.py new file mode 100644 index 00000000000..4791cd176b3 --- /dev/null +++ b/tools/aitools_quick_bench.py @@ -0,0 +1,340 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.12" +# /// +""" +Re-measure `./task aitools-quick`: its wall time after an edit, and how many of +the defects the full tier (`./task aitools-full`) catches it also catches. + + time Simulate an edit (append an unexported function to one production + file per aitools package), run aitools-quick, restore the file. + Reports p50/p95 wall time; --budget fails the run when p50 is over it. + detect Apply source mutants (flipped comparisons and booleans, swallowed + errors, removed calls, changed strings) one at a time, run the + quick tier on each and the full tier on the ones it misses, and + report the quick tier's share of the full tier's kills. Slow: about + a minute per mutant. + +Both modes edit files, so they run in a temporary git worktree of HEAD with the +checkout's uncommitted and untracked files copied in; the checkout itself is +never written to. + +Usage: + ./task aitools-quick-bench -- time --rounds 10 --budget 10 + ./task aitools-quick-bench -- time --cold + ./task aitools-quick-bench -- detect --sample 20 --seed 1 +""" + +import argparse +import json +import os +import random +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import uuid + +QUICK = ["uv", "run", "--script", "tools/aitools_quick.py"] +FULL = [*QUICK, "--full"] + +# One production file per aitools package: an edit there invalidates that +# package and everything that imports it. +EDIT_TARGETS = [ + "cmd/aitools/version.go", + "libs/aitools/agents/agents.go", + "libs/aitools/autoinstall/state.go", + "libs/aitools/installer/state.go", + "libs/aitools/mcp/state.go", + "libs/aitools/mcp/agentconfig/writer.go", + "libs/aitools/mcp/proxy/run.go", + "libs/aitools/mcpconfig/spec.go", + "experimental/aitools/cmd/aitools.go", + "experimental/aitools/lib/middlewares/warehouse.go", + "experimental/aitools/lib/session/session.go", +] + +MUTANT_DIRS = ["cmd/aitools", "libs/aitools", "experimental/aitools"] + + +def percentile(values, p): + """Nearest-rank percentile. + + >>> percentile([5, 1, 3, 2, 4], 50) + 3 + >>> percentile([5, 1, 3, 2, 4], 95) + 5 + """ + ordered = sorted(values) + rank = max(1, -(-p * len(ordered) // 100)) + return ordered[rank - 1] + + +def run_quick(env=None, steps=None): + """Run aitools-quick once; when steps is a dict, add each step's seconds to it.""" + with tempfile.TemporaryDirectory() as tmp: + report = os.path.join(tmp, "report.json") + start = time.monotonic() + proc = subprocess.run([*QUICK, "--json", report], capture_output=True, text=True, env=env) + seconds = time.monotonic() - start + if steps is not None and os.path.exists(report): + with open(report) as f: + for step in json.load(f)["steps"]: + steps.setdefault(step["step"], []).append(step["seconds"]) + return seconds, proc.returncode == 0, proc.stdout + proc.stderr + + +def simulate_edit(path): + """Append code that changes the package's compiled output, as a body edit would. + + The value differs on every call and a global store in init() can't be + optimized away, so the package and every test binary that links it are + rebuilt and re-run. The package's export data does not change, so + importers are relinked but not recompiled. The self-read keeps the unused + linter quiet. + """ + with open(path, "a") as f: + f.write( + "\nvar aitoolsQuickBenchSink string\n\n" + f'func init() {{\n\taitoolsQuickBenchSink = "{uuid.uuid4().hex}"\n\t_ = aitoolsQuickBenchSink\n}}\n' + ) + + +def cmd_time(args): + env = None + cache = None + if args.cold: + cache = tempfile.mkdtemp(prefix="aitools-quick-cold-") + env = dict(os.environ, GOCACHE=os.path.join(cache, "go"), GOLANGCI_LINT_CACHE=os.path.join(cache, "lint")) + try: + # Warm-up: the first run after a checkout or a cache wipe is the cold one. + seconds, ok, out = run_quick(env) + print(f"{'cold' if args.cold else 'warm-up'} run: {seconds:.2f}s ok={ok}") + if not ok: + print(out) + return 1 + if args.cold: + return 0 + times = {t: [] for t in EDIT_TARGETS} + steps = {} + for i in range(args.rounds): + for target in EDIT_TARGETS: + with open(target) as f: + original = f.read() + try: + simulate_edit(target) + seconds, ok, out = run_quick(steps=steps) + finally: + with open(target, "w") as f: + f.write(original) + if not ok: + print(f"aitools-quick failed after editing {target}:\n{out}") + return 1 + times[target].append(seconds) + print(f"round {i + 1} {target}: {seconds:.2f}s", flush=True) + finally: + if cache: + shutil.rmtree(cache, ignore_errors=True) + + every = [s for v in times.values() for s in v] + print(f"\n{'edited file':<52} {'p50':>6} {'p95':>6} {'max':>6}") + for target, v in times.items(): + print(f"{target:<52} {percentile(v, 50):6.2f} {percentile(v, 95):6.2f} {max(v):6.2f}") + p50, p95 = percentile(every, 50), percentile(every, 95) + print(f"{'all':<52} {p50:6.2f} {p95:6.2f} {max(every):6.2f} (n={len(every)})") + print(f"\n{'step (runs in parallel)':<52} {'p50':>6} {'p95':>6}") + for step, v in steps.items(): + print(f"{step:<52} {percentile(v, 50):6.2f} {percentile(v, 95):6.2f}") + if args.json: + with open(args.json, "w") as f: + json.dump({"p50": p50, "p95": p95, "runs": times, "steps": steps}, f, indent=2) + if args.budget and p50 > args.budget: + print(f"FAIL: p50 {p50:.2f}s is over the {args.budget}s budget") + return 1 + return 0 + + +def mask_strings(line): + """Blank out string literals and trailing comments so operators inside them are skipped. + + >>> mask_strings('a == "x == y" // b == c') + 'a == "______"' + """ + out = re.sub(r'"(?:[^"\\]|\\.)*"', lambda m: '"' + "_" * (len(m.group()) - 2) + '"', line) + out = re.sub(r"`[^`]*`", lambda m: "`" + "_" * (len(m.group()) - 2) + "`", out) + i = out.find("//") + return out if i < 0 else out[:i].rstrip() + + +# (name, pattern on the masked line, replacement for the matched text) +OPERATORS = [ + ("flip ==", r"==", "!="), + ("flip !=", r"!=", "=="), + ("flip &&", r"&&", "||"), + ("flip ||", r"\|\|", "&&"), + ("flip true", r"\btrue\b", "false"), + ("flip false", r"\bfalse\b", "true"), + ("swallow error", r"\berr$", "nil"), +] + + +def mutants(seed, sample): + found = [] + for root in MUTANT_DIRS: + for dirpath, dirnames, files in os.walk(root): + # Walk in a fixed order so a seed picks the same mutants on every machine. + dirnames.sort() + for name in sorted(files): + if not name.endswith(".go") or name.endswith("_test.go"): + continue + path = os.path.join(dirpath, name) + with open(path) as f: + lines = f.read().split("\n") + in_import = False + for n, line in enumerate(lines): + masked = mask_strings(line) + stripped = masked.strip() + # Mutants of import paths never compile. + if stripped == "import (": + in_import = True + if in_import: + in_import = stripped != ")" + continue + if not stripped or stripped.startswith(("import", "package")): + continue + for op, pattern, repl in OPERATORS: + if op == "swallow error" and not stripped.startswith("return"): + continue + for m in re.finditer(pattern, masked): + found.append((path, n, m.start(), m.end(), repl, op)) + # Drop a call statement such as cmd.AddCommand(x) or a flag registration. + if re.fullmatch(r"[\w.]+\(.*\)", stripped) and not stripped.startswith(("return", "defer", "go ")): + found.append((path, n, 0, len(line), "", "remove call")) + s = re.search(r'"[^"]+"', masked) + if s: + found.append((path, n, s.start() + 1, s.start() + 1, "x", "change string")) + random.Random(seed).shuffle(found) + return found[:sample] + + +def apply_mutant(path, n, start, end, repl): + with open(path) as f: + original = f.read() + lines = original.split("\n") + lines[n] = lines[n][:start] + repl + lines[n][end:] + with open(path, "w") as f: + f.write("\n".join(lines)) + return original + + +def cmd_detect(args): + for name, cmd in (("aitools-quick", QUICK), ("aitools-full", FULL)): + if subprocess.run(cmd, capture_output=True).returncode != 0: + print(f"{name} fails on the unmodified tree; fix that first") + return 1 + rows = [] + for path, n, start, end, repl, op in mutants(args.seed, args.sample): + original = apply_mutant(path, n, start, end, repl) + try: + if ( + subprocess.run( + ["go", "build", "./cmd/aitools/...", "./libs/aitools/...", "./experimental/aitools/..."], + capture_output=True, + ).returncode + != 0 + ): + continue # Does not compile: both tiers catch it trivially. + quick = subprocess.run(QUICK, capture_output=True).returncode != 0 + # The full tier runs everything the quick tier does, so it kills every mutant quick kills. + full = quick or subprocess.run(FULL, capture_output=True).returncode != 0 + finally: + with open(path, "w") as f: + f.write(original) + rows.append({"file": path, "line": n + 1, "op": op, "quick": quick, "full": full}) + print( + f"{path}:{n + 1} {op:<14} quick={'killed' if quick else 'missed'} full={'killed' if full else 'missed'}", + flush=True, + ) + full_kills = [r for r in rows if r["full"]] + both = [r for r in full_kills if r["quick"]] + print(f"\ncompiling mutants: {len(rows)}; killed by full: {len(full_kills)}; also killed by quick: {len(both)}") + if full_kills: + print(f"detection: {100 * len(both) / len(full_kills):.0f}%") + if args.json: + with open(args.json, "w") as f: + json.dump(rows, f, indent=2) + return 0 + + +def isolated_copy(root, copy): + """Create a worktree of HEAD at copy with the uncommitted and untracked files of the checkout at root.""" + subprocess.run(["git", "-C", root, "worktree", "add", "--quiet", "--detach", copy, "HEAD"], check=True) + # Pin the diff format: user config such as diff.noprefix or color.diff=always would break git apply. + diff = subprocess.run( + [ + "git", + "-C", + root, + "diff", + "--binary", + "--no-color", + "--no-ext-diff", + "--src-prefix=a/", + "--dst-prefix=b/", + "HEAD", + ], + capture_output=True, + check=True, + ).stdout + if diff: + subprocess.run(["git", "-C", copy, "apply", "--binary"], input=diff, check=True) + untracked = subprocess.run( + ["git", "-C", root, "ls-files", "-z", "--others", "--exclude-standard"], capture_output=True, check=True + ) + for path in untracked.stdout.decode().split("\0"): + if path: + os.makedirs(os.path.join(copy, os.path.dirname(path)), exist_ok=True) + shutil.copy2(os.path.join(root, path), os.path.join(copy, path), follow_symlinks=False) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="cmd", required=True) + t = sub.add_parser("time", help="measure wall time after simulated edits") + t.add_argument("--rounds", type=int, default=3) + t.add_argument("--budget", type=float, help="fail when the p50 wall time is over this many seconds") + t.add_argument("--cold", action="store_true", help="measure one run with empty Go and lint caches") + t.add_argument("--json", metavar="PATH") + d = sub.add_parser("detect", help="measure detection against the full tier") + d.add_argument("--sample", type=int, default=20) + d.add_argument("--seed", type=int, default=1) + d.add_argument("--json", metavar="PATH") + args = parser.parse_args() + if args.cmd == "time" and args.rounds < 1: + parser.error("--rounds must be at least 1") + if args.cmd == "time" and args.cold and (args.json or args.budget): + parser.error("--json and --budget do not apply to --cold, which measures a single run") + if args.json: + args.json = os.path.abspath(args.json) + + # Run the cleanup below on SIGTERM too, not only on Ctrl-C. + signal.signal(signal.SIGTERM, lambda *_: sys.exit(143)) + root = subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=True) + root = root.stdout.strip() + tmp = tempfile.mkdtemp(prefix="aitools-quick-bench-") + copy = os.path.join(tmp, "src") + try: + isolated_copy(root, copy) + os.chdir(copy) + return cmd_time(args) if args.cmd == "time" else cmd_detect(args) + finally: + os.chdir(root) + subprocess.run(["git", "worktree", "remove", "--force", copy], capture_output=True) + shutil.rmtree(tmp, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/lintdiff.py b/tools/lintdiff.py index 8090602b112..6f24e807980 100644 --- a/tools/lintdiff.py +++ b/tools/lintdiff.py @@ -46,6 +46,41 @@ def parse_lines(cmd): return [x.strip() for x in result.stdout.split("\n") if x.strip()] +def go_dirs(paths, filter_nested): + """Return ./dir patterns for the directories of the Go files among paths. + + Skips testdata and, with filter_nested, nested modules. Keeps only + directories that still contain a Go file, so a package whose last Go file + was deleted is dropped. + """ + dirs = set() + for filename in paths: + if "/testdata/" in filename: + continue + if filename.endswith(".go"): + d = os.path.dirname(filename) + if filter_nested and in_nested_module(d): + continue + dirs.add(d) + + return ["./" + d for d in sorted(dirs) if os.path.isdir(d) and any(n.endswith(".go") for n in os.listdir(d))] + + +def changed_go_dirs(ref, filter_nested=True): + """Return ./dir patterns for the Go packages changed since the merge base with ref, or None if git diff fails. + + Diff the merge base against the working tree so we only see files changed + on this branch, not files that advanced on the ref since the branch point. + Unlike testmask's three-dot "ref...HEAD" form (tools/testmask/git.go), this + includes the dirty tree's uncommitted changes. --no-renames lists both sides + of a rename, so the package a file moved out of is selected too. + """ + changed = parse_lines(["git", "diff", "--name-only", "--no-renames", "--merge-base", ref, "--", "."]) + if changed is None: + return None + return go_dirs(changed, filter_nested) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--ref", default="main", help="Reference to calculate diff against.") @@ -66,12 +101,6 @@ def main(): if gitroot: os.chdir(gitroot[0]) - # Diff the merge base against the working tree so we only lint files changed - # on this branch, not files that advanced on the ref since the branch point. - # Unlike testmask's three-dot "ref...HEAD" form (tools/testmask/git.go), this - # includes the dirty tree's uncommitted changes, which is the point of --fix. - changed = parse_lines(["git", "diff", "--name-only", "--merge-base", args.ref, "--", "."]) - cmd = args.args[:] # `golangci-lint run` typechecks against the target go.mod and errors on @@ -79,21 +108,12 @@ def main(): # cross-module safe. Apply the nested-module filter only for `run`. filter_nested = "run" in cmd - if changed is not None: - # We need to pass packages to golangci-lint, not individual files. - # QQQ for lint we should also pass all dependent packages - dirs = set() - for filename in changed: - if "/testdata/" in filename: - continue - if filename.endswith(".go"): - d = os.path.dirname(filename) - if filter_nested and in_nested_module(d): - continue - dirs.add(d) - - dirs = ["./" + d for d in sorted(dirs) if os.path.exists(d)] + # The diff includes the dirty tree's uncommitted changes, which is the point of --fix. + # We need to pass packages to golangci-lint, not individual files. + # QQQ for lint we should also pass all dependent packages + dirs = changed_go_dirs(args.ref, filter_nested) + if dirs is not None: if not dirs: sys.exit(0)