diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index bc82b6b..d7142ab 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -40,6 +40,7 @@ jobs: with: python-version: "3.12" # Org block reads need an App token; GITHUB_TOKEN can't see them. + # actions: read lists recent health checks, so a dead report doesn't queue a repeat. - id: token uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 with: @@ -47,7 +48,9 @@ jobs: private-key: ${{ secrets.APP_PRIVATE_KEY }} permission-organization-user-blocking: read permission-issues: read + permission-actions: read # An API failure exits non-zero with no result file, so nothing is acted on. + # It also writes undecided.json, counted below; a rate limit that hasn't reset yet writes nothing. # A dispatched issue that is already handled, or a bot's issue, exits 0 with no result file. - env: GITHUB_TOKEN: ${{ steps.token.outputs.token }} @@ -55,9 +58,9 @@ jobs: ISSUE_INPUT: ${{ inputs.issue }} run: | if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then - python -m scripts.reports decide --issue "$ISSUE_INPUT" --out result.json + python -m scripts.reports decide --issue "$ISSUE_INPUT" --out result.json --undecided undecided.json else - python -m scripts.reports decide --event "$GITHUB_EVENT_PATH" --out result.json + python -m scripts.reports decide --event "$GITHUB_EVENT_PATH" --out result.json --undecided undecided.json fi # Its own token, so the one that reads issue text never carries actions: write. - id: dispatch_token @@ -71,7 +74,7 @@ jobs: - if: hashFiles('result.json') != '' env: GITHUB_TOKEN: ${{ steps.dispatch_token.outputs.token }} - run: python -m scripts.reports dispatch --result result.json + run: python -m scripts.reports dispatch --result result.json --undecided undecided.json - if: hashFiles('result.json') != '' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -87,6 +90,18 @@ jobs: print("has=" + ("true" if r["actions"] else "false")) print("hide=" + ("true" if any(a["kind"] == "hide" for a in r["actions"]) else "false")) PY + # After 3 runs without a decision the report is parked as needs-review, so the sweeper moves on. + - id: undecided_token + if: failure() && hashFiles('undecided.json') != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-issues: write + - if: failure() && steps.undecided_token.outputs.token != '' + env: + GITHUB_TOKEN: ${{ steps.undecided_token.outputs.token }} + run: python -m scripts.reports undecided --file undecided.json hide: needs: decide diff --git a/.github/workflows/rot.yml b/.github/workflows/rot.yml index 7511d20..6a0d46b 100644 --- a/.github/workflows/rot.yml +++ b/.github/workflows/rot.yml @@ -1,6 +1,7 @@ name: Health check # The sweeper only counts runs named exactly "Health check" as a full check. -run-name: ${{ inputs.repo && 'Health check (one repo)' || 'Health check' }} +# A one-repo run names its repo, so reports.py can see a recent check of that entry. +run-name: "${{ inputs.repo && format('Health check: {0}', inputs.repo) || 'Health check' }}" # Re-checks every listed repo each Monday: warns stale ones, archives them 30 # days later, brings back the ones that recover, follows renames and removes @@ -54,6 +55,21 @@ jobs: GITHUB_TOKEN: ${{ steps.token.outputs.token }} REPO_INPUT: ${{ inputs.repo }} run: python -m scripts.rot plan --out "$RUNNER_TEMP/plan.json" >> "$GITHUB_OUTPUT" + # Entries left undecided 3 weeks running get a needs-review issue. Its own token, so the one + # that can push never carries issues: write, and a failure here never stops the check. + - id: issues_token + if: steps.plan.outputs.review == 'true' + continue-on-error: true + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-issues: write + - if: steps.issues_token.outputs.token != '' + continue-on-error: true + env: + GITHUB_TOKEN: ${{ steps.issues_token.outputs.token }} + run: python -m scripts.rot review --plan "$RUNNER_TEMP/plan.json" - id: pr if: steps.plan.outputs.changed == 'true' env: diff --git a/.github/workflows/submit.yml b/.github/workflows/submit.yml index 8fa96cb..8f78d60 100644 --- a/.github/workflows/submit.yml +++ b/.github/workflows/submit.yml @@ -47,6 +47,8 @@ jobs: permission-organization-user-blocking: read permission-issues: read # An API failure exits non-zero with no result file, so nothing is accepted or rejected. + # Any failure (an undecidable repo, an API error) also writes undecided.json, counted below. + # A rate limit that hasn't reset yet writes nothing, so waiting for it spends no attempt. # A dispatched issue that is already decided exits 0 with no result file, so nothing runs. - env: GITHUB_TOKEN: ${{ steps.token.outputs.token }} @@ -54,10 +56,22 @@ jobs: ISSUE_INPUT: ${{ inputs.issue }} run: | if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then - python -m scripts.submit decide --issue "$ISSUE_INPUT" --out result.json + python -m scripts.submit decide --issue "$ISSUE_INPUT" --out result.json --undecided undecided.json else - python -m scripts.submit decide --event "$GITHUB_EVENT_PATH" --out result.json + python -m scripts.submit decide --event "$GITHUB_EVENT_PATH" --out result.json --undecided undecided.json fi + # After 3 runs without a decision the issue is parked as needs-review, so the sweeper moves on. + - id: undecided_token + if: failure() && hashFiles('undecided.json') != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-issues: write + - if: failure() && steps.undecided_token.outputs.token != '' + env: + GITHUB_TOKEN: ${{ steps.undecided_token.outputs.token }} + run: python -m scripts.submit undecided --file undecided.json - if: hashFiles('result.json') != '' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: diff --git a/scripts/devstream/attempts.py b/scripts/devstream/attempts.py new file mode 100644 index 0000000..59c1314 --- /dev/null +++ b/scripts/devstream/attempts.py @@ -0,0 +1,59 @@ +"""Bound how often an issue's decide run can fail, so one stuck issue can't hold the sweeper.""" + +import json +import re +import time + +from scripts.devstream.gh import RateLimited + +API = "/repos/devstreamhq/devstream" +BOT = "devstreamhq-bot[bot]" +MAX_FAILURES = 3 +COUNTER = re.compile(r"") +RETRYING = "We couldn't finish checking this one yet, so we'll try again shortly.\n\n\n" +PARKED = "We couldn't finish checking this one, so it's waiting for a maintainer.\n\n\n" + + +def spends_attempt(exc: BaseException) -> bool: + """A rate limit still in force just waits for its reset; any other failure counts.""" + return not (isinstance(exc, RateLimited) and exc.reset_at > time.time()) + + +def note_failure(path: str, issue: int, exc: BaseException) -> None: + with open(path, "w", encoding="utf-8") as f: + json.dump({"issue": issue, "error": str(exc)}, f) + + +def record_failure(gh, n: int, error: str, error_body: str) -> bool: + """Count one more failed run on issue n; from the MAX_FAILURES-th on, park it. True when parked. + + The count lives in a hidden marker in the bot's own comment, so the issue's author can't forge it. + """ + mine = next((c for c in gh.paginate(f"{API}/issues/{n}/comments?per_page=100") + if (c.get("user") or {}).get("login") == BOT and COUNTER.search(c.get("body") or "")), None) + # Read-then-write, so two runs failing at once can count as one. That's fine: the sweeper + # dispatches one item at a time, and a lost count only costs one extra retry. + count = int(COUNTER.search(mine["body"]).group(1)) + 1 if mine else 1 + body = (RETRYING if count < MAX_FAILURES else PARKED).format(count) + if mine: + gh.patch(f"{API}/issues/comments/{mine['id']}", {"body": body}) + else: + gh.post(f"{API}/issues/{n}/comments", {"body": body}) + if count < MAX_FAILURES: + return False + # The sweeper skips needs-review, so it moves on. Re-added each time, so a maintainer's retry + # that fails again can't bring the starvation back. + gh.post(f"{API}/issues/{n}/labels", {"labels": ["needs-review"]}) + if count > MAX_FAILURES: + return True + detail = " ".join(error.replace("`", "'").split())[:300] + gh.post(f"{API}/issues/{n}/comments", {"body": error_body + f"\n`{detail}`\n"}) + return True + + +def record_from_file(gh, path: str, error_body: str) -> None: + with open(path, encoding="utf-8") as f: + u = json.load(f) + if type(u["issue"]) is not int or u["issue"] <= 0: + raise SystemExit(f"bad issue number {u['issue']!r}") + record_failure(gh, u["issue"], str(u["error"]), error_body) diff --git a/scripts/devstream/gh.py b/scripts/devstream/gh.py index fbcc3c7..0a2e9bb 100644 --- a/scripts/devstream/gh.py +++ b/scripts/devstream/gh.py @@ -14,6 +14,9 @@ ATTEMPTS = 3 TRANSIENT = (502, 503, 504) NETWORK = (urllib.error.URLError, TimeoutError, ConnectionError) +# Only a document that opens as a query is a read; anything else (a fragment first, say) may be a mutation. +LEADING_COMMENTS = re.compile(r"(\s*#[^\n]*(\n|$))*") +READ_QUERY = re.compile(r"\s*(query\b|\{)") class GitHubError(Exception): @@ -185,7 +188,7 @@ def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> """allow_not_found returns data with nulls where only NOT_FOUND errors occurred.""" self.last_redirect = None r = self._request("POST", "/graphql", body={"query": query, "variables": variables}, - read=not query.lstrip().startswith("mutation")) + read=bool(READ_QUERY.match(query, LEADING_COMMENTS.match(query).end()))) if r is None: raise GitHubError(404, "graphql endpoint not found") out = r[1] or {} diff --git a/scripts/devstream/license_detect.py b/scripts/devstream/license_detect.py index 642f728..7275b69 100644 --- a/scripts/devstream/license_detect.py +++ b/scripts/devstream/license_detect.py @@ -25,6 +25,8 @@ def match(name): FILE_RE = _FileRe() MAX_FILES = 5 MAX_BYTES = 100_000 +# The blob comes down whole before any cap, so nothing larger is fetched at all. +MAX_BLOB = 5_000_000 NON_OSI = [ ("Server Side Public License", "Server Side Public License"), @@ -106,6 +108,10 @@ class Undecided(GitHubError): pass +class TooLarge(Undecided): + pass + + def _norm(text: str) -> str: return re.sub(r"\s+", " ", re.sub("[\"\u201c\u201d]", "", text)).lower() @@ -258,16 +264,32 @@ def _readme_license(readme: str, allowed: frozenset = frozenset()) -> tuple[str return None, None +def file_bytes(gh, r, data: dict, what: str) -> bytes: + """The bytes of a contents-API file. Undecided only when the content can't be decoded.""" + if data.get("encoding") == "none" and data.get("sha"): + if (data.get("size") or 0) > MAX_BLOB: + raise TooLarge(0, f"{what} of {r} is over {MAX_BLOB // 1_000_000} MB") + # Over 1 MB the contents API leaves content empty; the blobs API serves up to 100 MB. + data = gh.get(f"/repos/{r}/git/blobs/{data['sha']}") + if data is None: + raise GitHubError(404, f"blob for {what} of {r} not found") + if data.get("encoding") != "base64": + raise Undecided(0, f"{what} of {r} has no readable content") + try: + return base64.b64decode("".join((data.get("content") or "").split()), validate=False) + except (binascii.Error, ValueError) as e: + raise Undecided(0, f"{what} of {r} could not be decoded: {e}") from e + + def _read_file(gh, r, path) -> str: data = gh.get(f"/repos/{r}/contents/{path}") if not data: return "" - if data.get("encoding") != "base64": - raise Undecided(0, f"{path} of {r} is too large to read") try: - raw = base64.b64decode("".join((data.get("content") or "").split()), validate=False) - except (binascii.Error, ValueError) as e: - raise Undecided(0, f"{path} of {r} could not be decoded: {e}") from e + raw = file_bytes(gh, r, data, path) + except TooLarge: + # No license text, so the rule fails closed; rot's second sighting still guards a removal. + return "" return raw[:MAX_BYTES].decode("utf-8", errors="replace") diff --git a/scripts/devstream/quality.py b/scripts/devstream/quality.py index 1e34e8b..596f077 100644 --- a/scripts/devstream/quality.py +++ b/scripts/devstream/quality.py @@ -1,5 +1,3 @@ -import base64 -import binascii import json import re from dataclasses import dataclass, field @@ -8,7 +6,7 @@ from typing import Literal from scripts.devstream.gh import GitHubError -from scripts.devstream.license_detect import OSI, Undecided, find_license +from scripts.devstream.license_detect import OSI, Undecided, file_bytes, find_license NO_LICENSE = {None, "NOASSERTION", "NONE"} DEPRECATED = re.compile(r"\b(deprecated|unmaintained|no longer maintained)\b", re.I) @@ -87,18 +85,12 @@ def _license(repo, gh, r, readme_text, allowed=frozenset(), mode="submit") -> st return MSG_NOT_FOUND_CC if cc_only and mode == "submit" else MSG_NOT_FOUND -def _readme(gh, r) -> tuple[str | None, int] | None: - """None for no README, else (text, length). text is None when GitHub withholds it (over 1 MB).""" +def _readme(gh, r) -> tuple[str, int] | None: + """None for no README, else (text, length).""" data = gh.get(f"/repos/{r}/readme") if data is None: return None - if data.get("encoding") == "none": - return None, data.get("size") or 0 - try: - raw = base64.b64decode("".join((data.get("content") or "").split()), validate=False) - except (binascii.Error, ValueError) as e: - raise Undecided(0, f"README of {r} could not be decoded: {e}") from e - text = raw.decode("utf-8", errors="replace")[:README_SCAN] + text = file_bytes(gh, r, data, "README").decode("utf-8", errors="replace")[:README_SCAN] return text, _readme_text_length(text) diff --git a/scripts/reports.py b/scripts/reports.py index 28c6a03..8cb42c2 100644 --- a/scripts/reports.py +++ b/scripts/reports.py @@ -5,6 +5,8 @@ python -m scripts.reports apply --result result.json python -m scripts.reports issue --result result.json [--error] python -m scripts.reports pr-body --result result.json +python -m scripts.reports decide ... --undecided undecided.json (exits 3 when the run reached no decision) +python -m scripts.reports undecided --file undecided.json (counts it, and parks the report after 3) """ import argparse @@ -12,9 +14,12 @@ import os import re import sys +import traceback +import urllib.parse from dataclasses import asdict, dataclass from datetime import datetime, timedelta, timezone +from scripts.devstream.attempts import note_failure, record_from_file, spends_attempt from scripts.devstream.entries import DEFAULT_PATH, find, load, normalize_repo, save from scripts.devstream.gh import GitHub from scripts.devstream.issue_form import FormError, _repo, parse_report @@ -30,6 +35,11 @@ ESTABLISHED_AGE = timedelta(days=365) ESTABLISHED_REPOS = 3 ESTABLISHED_CONTRIBUTIONS = 50 +# rot.yml names a one-repo run "Health check: owner/name". +ONE_REPO = "Health check: " +RECHECK_AFTER = timedelta(hours=24) +# One-repo checks an hour, across all reporters, before new dead reports wait for a maintainer. +CHECKS_PER_HOUR = 3 NOT_LISTED = "That repo isn't on the list, so there's nothing here for us to act on." UNTRUSTED = ( @@ -86,6 +96,20 @@ def _other_recent_reports(login: str, number: int, gh, now: datetime) -> int: return sum(1 for i in gh.get(f"/search/issues?q={q}&per_page=100")["items"] if i["number"] != number) +def _recent_checks(gh, now: datetime) -> list[tuple[datetime, str]]: + """(created, normalised repo) for one-repo health checks from the last 24 hours, any status. + + Every queued or running check is in there too: a run can't wait in the queue longer than that. + """ + since = (now - RECHECK_AFTER).strftime("%Y-%m-%dT%H:%M:%SZ") + query = urllib.parse.urlencode({"created": f">={since}", "per_page": 100}) + # shortcut: newest 100 runs only. The cap allows 72 a day, plus manual runs; paginate if those + # ever push a day's one-repo checks past 100. + page = gh.get(f"{API}/actions/workflows/rot.yml/runs?{query}") or {} + return [(datetime.fromisoformat(r["created_at"].replace("Z", "+00:00")), normalize_repo(r["display_title"][len(ONE_REPO):])) + for r in page.get("workflow_runs", []) if (r.get("display_title") or "").startswith(ONE_REPO)] + + def _unedited(gh, numbers: list[int]) -> set[int]: """The issues whose body is still as filed: an edited report could have been decided as something else.""" if not numbers: @@ -149,6 +173,13 @@ def handle_report(issue: dict, gh, entries: list, open_reports: list[dict], hidden = f"Thanks for the heads-up. `{repo}` is hidden from the list while a maintainer reviews it." if reason == "dead": + recent = _recent_checks(gh, now) + if normalize_repo(repo) in {r for _, r in recent}: + return _reply(f"Thanks. `{repo}` had a health check in the last day, " + "so a maintainer will take it from here.", "needs-review") + if sum(now - at <= timedelta(hours=1) for at, _ in recent) >= CHECKS_PER_HOUR: + return _reply(f"Thanks for reporting `{repo}`. A lot of health checks are queued right now, " + "so a maintainer will take a look instead.", "needs-review") return [Action("dispatch_rot", repo)] + _reply( f"A health check on `{repo}` has been queued.", "needs-review") @@ -205,6 +236,24 @@ def _values(result: dict, kind: str) -> list: def _decide(args) -> int: + """Exit 3 with undecided.json on any failure, so the report can be parked after a few.""" + try: + return _decide_issue(args) + except Exception as e: + if args.undecided is None or not spends_attempt(e): + raise + traceback.print_exc() + n = int(args.issue) if args.issue is not None else _read(args.event)["issue"]["number"] + note_failure(args.undecided, n, e) + return 3 + + +def _undecided(args) -> int: + record_from_file(GitHub(os.environ["GITHUB_TOKEN"]), args.file, ERROR) + return 0 + + +def _decide_issue(args) -> int: gh = GitHub(os.environ["GITHUB_TOKEN"]) if args.issue is not None: issue = _fetch_issue(gh, args.issue) @@ -224,9 +273,18 @@ def _decide(args) -> int: def _dispatch(args) -> int: - for repo in _values(_read(args.result), "dispatch_rot"): - GitHub(os.environ["GITHUB_TOKEN"]).post( - f"{API}/actions/workflows/rot.yml/dispatches", {"ref": "main", "inputs": {"repo": repo}}) + result = _read(args.result) + try: + for repo in _values(result, "dispatch_rot"): + GitHub(os.environ["GITHUB_TOKEN"]).post( + f"{API}/actions/workflows/rot.yml/dispatches", {"ref": "main", "inputs": {"repo": repo}}) + except Exception as e: + # No comment has gone out yet, so this run reached no decision either. + if args.undecided is None or not spends_attempt(e): + raise + traceback.print_exc() + note_failure(args.undecided, result["issue"], e) + return 3 return 0 @@ -284,14 +342,17 @@ def main(argv: list[str]) -> int: src.add_argument("--event") src.add_argument("--issue") d.add_argument("--out", required=True) + d.add_argument("--undecided") + sub.add_parser("undecided").add_argument("--file", required=True) for name in ("dispatch", "apply", "issue", "pr-body"): s = sub.add_parser(name) s.add_argument("--result", required=True) sub.choices["apply"].add_argument("--entries", default=DEFAULT_PATH) sub.choices["issue"].add_argument("--error", action="store_true") + sub.choices["dispatch"].add_argument("--undecided") args = p.parse_args(argv) return {"decide": _decide, "dispatch": _dispatch, "apply": _apply, "issue": _issue, - "pr-body": _pr_body}[args.cmd](args) + "pr-body": _pr_body, "undecided": _undecided}[args.cmd](args) if __name__ == "__main__": diff --git a/scripts/rot.py b/scripts/rot.py index 4525bca..bd55bda 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -3,6 +3,7 @@ python -m scripts.rot plan --out plan.json (prints changed=true|false; REPO_INPUT limits it to one entry) python -m scripts.rot body --plan plan.json python -m scripts.rot finish --plan plan.json (only after the entries PR has merged) +python -m scripts.rot review --plan plan.json (opens needs-review issues for entries undecided 3 weeks running) """ import argparse @@ -38,6 +39,15 @@ RENAME_KEEP = timedelta(days=8) REMOVAL_REASONS = (MSG_GONE, MSG_LICENSE, MSG_NO_README, MSG_SHORT_README) REPO_INPUT = re.compile(r"[A-Za-z0-9-]{1,39}/[A-Za-z0-9._-]{1,100}") +API = "/repos/devstreamhq/devstream" +BOT = "devstreamhq-bot[bot]" +# Full runs in a row that couldn't decide on an entry before a maintainer is asked to look. +UNDECIDED_WEEKS = 3 +REVIEW_TITLE = "Health check can't decide on {repo}" +REVIEW_BODY = ( + "The weekly health check couldn't decide on `{repo}` for {weeks} weeks running, " + "so it isn't being warned or removed. Could someone take a look?\n\nLast error: `{error}`\n" +) WARN = ( "Heads up: `{repo}` moves to the archive on {date} unless something changes. " @@ -200,11 +210,20 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: rot_state = read_json(gh, "rot.json")[0] rot_state = rot_state or {} results = {} + undecided = {} for e in targets: try: results[e["repo"]] = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"])) except Undecided as exc: logging.warning("skipping %s: %s", e["repo"], exc) + undecided[normalize_repo(e["repo"])] = str(exc) + review = [] + if repo is None: + try: + review = _track_undecided(gh, now, {normalize_repo(e["repo"]) for e in targets}, undecided) + except Exception: + # The weeks are only a review prompt; failing to record them mustn't stop the check. + logging.exception("could not record undecided repos") new, actions, rot = plan_rot(entries, results, now, rot_state) changed = new != sorted(entries, key=sort_key) if changed: @@ -221,7 +240,39 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: failed += _mark_notified(gh, notified, rot) if failed: raise SystemExit(f"{failed} Discussion or stats update(s) failed") - return {"changed": changed, "actions": [asdict(a) for a in actions], "rot_state": rot} + return {"changed": changed, "actions": [asdict(a) for a in actions], "rot_state": rot, "review": review} + + +def _track_undecided(gh, now, checked: set[str], undecided: dict[str, str]) -> list[dict]: + """Count full runs in a row that skipped each entry. Returns the entries due a review issue.""" + def mutate(data): + data = {k: v for k, v in data.items() if k in checked and k in undecided} + for k in undecided: + rec = data.get(k) + # A retried run in the same week isn't another week. + if rec is None or now - _parse(rec["at"]) >= CONFIRM: + data[k] = {"weeks": (rec or {}).get("weeks", 0) + 1, "at": _iso(now)} + return data + + update_json(gh, "undecided.json", mutate, "chore(stats): record undecided repos") + data = read_json(gh, "undecided.json")[0] or {} + return [{"repo": k, "weeks": v["weeks"], "error": " ".join(undecided[k].replace("`", "'").split())[:300]} + for k, v in sorted(data.items()) if v["weeks"] >= UNDECIDED_WEEKS and k in undecided] + + +def open_reviews(gh, review: list[dict]) -> None: + """One needs-review issue per entry, unless the bot already has one open with that title.""" + if not review: + return + open_titles = {i["title"] for i in gh.paginate(f"{API}/issues?state=open&labels=needs-review&per_page=100") + if (i.get("user") or {}).get("login") == BOT} + for r in review: + if not REPO_INPUT.fullmatch(r["repo"]): + raise ValueError(f"not an owner/name: {r['repo']!r}") + title = REVIEW_TITLE.format(repo=r["repo"]) + if title in open_titles: + continue + gh.post(f"{API}/issues", {"title": title, "body": REVIEW_BODY.format(**r), "labels": ["needs-review"]}) def pr_body(actions) -> str: @@ -353,7 +404,7 @@ def main(argv: list[str]) -> int: pl = sub.add_parser("plan") pl.add_argument("--out", required=True) pl.add_argument("--entries", default=DEFAULT_PATH) - for name in ("body", "finish"): + for name in ("body", "finish", "review"): sub.add_parser(name).add_argument("--plan", required=True) args = p.parse_args(argv) @@ -366,10 +417,15 @@ def main(argv: list[str]) -> int: with open(args.plan, encoding="utf-8") as f: finish(gh, json.load(f)) return 0 + if args.cmd == "review": + with open(args.plan, encoding="utf-8") as f: + open_reviews(gh, json.load(f)["review"]) + return 0 plan = prepare(gh, datetime.now(timezone.utc), args.entries, os.environ.get("REPO_INPUT") or None) with open(args.out, "w", encoding="utf-8") as f: json.dump(plan, f, indent=2) print(f"changed={'true' if plan['changed'] else 'false'}") + print(f"review={'true' if plan['review'] else 'false'}") return 0 diff --git a/scripts/submit.py b/scripts/submit.py index 6f092c5..78e1afa 100644 --- a/scripts/submit.py +++ b/scripts/submit.py @@ -2,6 +2,8 @@ python -m scripts.submit decide (--event EVENT.json | --issue N) --out result.json python -m scripts.submit apply --result result.json +python -m scripts.submit decide ... --undecided undecided.json (exits 3 when the run reached no decision) +python -m scripts.submit undecided --file undecided.json (counts it, and parks the issue after 3) python -m scripts.submit comment --result result.json --kind {accepted,rejected,pr,error} python -m scripts.submit seed --file seeds.csv --out report.json [--apply] """ @@ -12,9 +14,11 @@ import os import re import sys +import traceback from dataclasses import asdict, dataclass, field from datetime import datetime, timedelta, timezone +from scripts.devstream.attempts import note_failure, record_from_file, spends_attempt from scripts.devstream.description import clean_description from scripts.devstream.entries import CATEGORIES, DEFAULT_PATH, Entry, find, load, normalize_repo, save from scripts.devstream.gh import GitHub, GitHubError, RateLimited @@ -165,7 +169,29 @@ def _fetch_issue(gh, raw: str) -> dict | None: return issue +def _undecided(args) -> int: + record_from_file(GitHub(os.environ["GITHUB_TOKEN"]), args.file, comment({}, "error")) + return 0 + + def _decide(args) -> int: + """Exit 3 with undecided.json on any failure, so the issue can be parked after a few.""" + try: + return _decide_issue(args) + except Exception as e: + if args.undecided is None or not spends_attempt(e): + raise + traceback.print_exc() + if args.issue is not None: + n = int(args.issue) + else: + with open(args.event, encoding="utf-8") as f: + n = json.load(f)["issue"]["number"] + note_failure(args.undecided, n, e) + return 3 + + +def _decide_issue(args) -> int: gh = GitHub(os.environ["GITHUB_TOKEN"]) if args.issue is not None: issue = _fetch_issue(gh, args.issue) @@ -278,6 +304,9 @@ def main(argv: list[str]) -> int: src.add_argument("--event") src.add_argument("--issue") d.add_argument("--out", required=True) + d.add_argument("--undecided") + u = sub.add_parser("undecided") + u.add_argument("--file", required=True) a = sub.add_parser("apply") a.add_argument("--result", required=True) a.add_argument("--entries", default=DEFAULT_PATH) @@ -290,7 +319,8 @@ def main(argv: list[str]) -> int: c.add_argument("--result", required=True) c.add_argument("--kind", required=True, choices=("accepted", "rejected", "pr", "error")) args = p.parse_args(argv) - return {"decide": _decide, "apply": _apply, "comment": _comment, "seed": _seed}[args.cmd](args) + return {"decide": _decide, "apply": _apply, "comment": _comment, "seed": _seed, + "undecided": _undecided}[args.cmd](args) if __name__ == "__main__": diff --git a/scripts/sweep.py b/scripts/sweep.py index e2f5f1a..1964f1e 100644 --- a/scripts/sweep.py +++ b/scripts/sweep.py @@ -20,7 +20,7 @@ # Each failed rot run can leave a PR behind, so a failing check is retried at most daily. ROT_RETRY = timedelta(days=1) ROT_BRANCH = re.compile(r"rot/[0-9]{1,20}") -# rot.yml's run-name for a full run; a one-repo re-check doesn't cover the list. +# rot.yml's run-name for a full run; a one-repo re-check ("Health check: owner/name") doesn't cover the list. FULL_ROT = "Health check" diff --git a/scripts/tests/test_gh.py b/scripts/tests/test_gh.py index 07f11b8..9376eed 100644 --- a/scripts/tests/test_gh.py +++ b/scripts/tests/test_gh.py @@ -344,6 +344,24 @@ def test_graphql_mutation_502_not_retried(self): gh.graphql("mutation M { x }", {}) self.assertEqual(len(sent), 1) + def test_graphql_read_only_when_it_opens_with_query_or_brace(self): + cases = ( + ("query Q { x }", True), ("{ viewer }", True), (" \n{ viewer }", True), + ("# note\n query Q { x }", True), ("# a\n# b\n{ x }", True), + # A comment line used to hide a mutation from the read check. + ("# note\nmutation M { x }", False), + # Fragment-first could hold a mutation further down; treated as one. + ("fragment F on X { y }\nquery Q { ...F }", False), + ("mutation M { x }", False), ("subscription S { x }", False), ("queryish { x }", False), + ) + for q, read in cases: + gh, sent, _ = self.gh(502, 200) + try: + gh.graphql(q, {}) + except GitHubError: + pass + self.assertEqual(len(sent), 2 if read else 1, q) + def test_graphql_mutation_retried_when_never_sent(self): gh, sent, _ = self.gh(urllib.error.URLError(ConnectionRefusedError()), 200) gh.graphql("\n mutation M { x }", {}) diff --git a/scripts/tests/test_license_detect.py b/scripts/tests/test_license_detect.py index 8e53502..d821593 100644 --- a/scripts/tests/test_license_detect.py +++ b/scripts/tests/test_license_detect.py @@ -254,9 +254,45 @@ def test_symlink_counts_as_present_and_is_not_read(self): self.assertEqual(run(g).failures, [MSG_NOT_FOUND]) self.assertFalse([c for c in g.calls if c[1] == f"{R}/contents/LICENSE"]) - def test_oversize_file_is_undecided(self): + def oversize(self, text): g = lic_gh({"LICENSE": MIT}) - g.routes[f"GET {R}/contents/LICENSE"] = {"encoding": "none", "size": 2_000_000} + g.routes[f"GET {R}/contents/LICENSE"] = {"type": "file", "encoding": "none", "size": len(text), "sha": "big"} + g.routes[f"GET {R}/git/blobs/big"] = {"encoding": "base64", "sha": "big", + "content": base64.encodebytes(text.encode()).decode()} + return g + + def test_oversize_file_is_read_through_the_blob(self): + busl = "Business Source License 1.1\n\n" + "Licensed Work terms. " * 100_000 + self.assertGreater(len(busl), 2_000_000) + g = self.oversize(busl) + self.assertEqual(run(g).failures, [MSG_NON_OSI.format("Business Source License")]) + self.assertEqual(run(g, "rot").rot, "remove") + self.assertIn(("GET", f"{R}/git/blobs/big", None), g.calls) + + def test_oversize_file_is_capped_at_the_scan_limit(self): + # Only the first 100 KB is scanned, the same as a file under 1 MB. + self.assertEqual(run(self.oversize("x" * 150_000 + MIT)).failures, [MSG_NOT_FOUND]) + self.assertEqual(run(self.oversize(MIT + "\n" + "x" * 2_000_000)).failures, []) + + def test_file_over_5mb_is_never_downloaded_and_counts_as_no_license(self): + g = lic_gh({"LICENSE": MIT}, readme=README + "\n## License\n\nMIT License\n") + g.routes[f"GET {R}/contents/"] = [{"name": "LICENSE", "type": "file", "size": 100_000_000}] + g.routes[f"GET {R}/contents/LICENSE"] = {"type": "file", "encoding": "none", "size": 100_000_000, "sha": "big"} + g.routes[f"GET {R}/git/blobs/big"] = {"encoding": "base64", "sha": "big", + "content": base64.b64encode(MIT.encode()).decode()} + # Fails closed: the file is present, so the README's claim doesn't stand in for it. + self.assertEqual(run(g).failures, [MSG_NOT_FOUND]) + self.assertEqual(run(g, "rot").rot, "remove") + self.assertFalse([c for c in g.calls if "/git/blobs/" in c[1]]) + + def test_3mb_file_is_still_read_through_the_blob(self): + g = self.oversize("Business Source License 1.1\n\n" + "x" * 3_000_000) + self.assertEqual(run(g).failures, [MSG_NON_OSI.format("Business Source License")]) + self.assertIn(("GET", f"{R}/git/blobs/big", None), g.calls) + + def test_oversize_file_without_a_blob_sha_is_undecided(self): + g = lic_gh({"LICENSE": MIT}) + g.routes[f"GET {R}/contents/LICENSE"] = {"type": "file", "encoding": "none", "size": 2_000_000} with self.assertRaises(Undecided): run(g) diff --git a/scripts/tests/test_quality.py b/scripts/tests/test_quality.py index 22e63ab..e5f2b52 100644 --- a/scripts/tests/test_quality.py +++ b/scripts/tests/test_quality.py @@ -313,11 +313,38 @@ def test_deprecated_message_names_the_word(self): self.assertEqual(len(r.failures), 1) self.assertIn('"Deprecated"', r.failures[0]) - def test_readme_encoding_none_uses_size(self): - big = {"encoding": "none", "content": "", "size": 2_000_000} - self.assertEqual(run(gh(extra={f"GET {R}/readme": big})).failures, []) - small = {"encoding": "none", "content": "", "size": 100} - self.assertEqual(run(gh(extra={f"GET {R}/readme": small})).failures, [MSG_SHORT]) + def test_readme_over_1mb_is_read_through_the_blob(self): + def big(text): + return {f"GET {R}/readme": {"encoding": "none", "content": "", "size": len(text), "sha": "big"}, + f"GET {R}/git/blobs/big": {"encoding": "base64", "sha": "big", "content": b64(text)["content"]}} + + # 2 MB of headings is measured as empty, not passed on its size. + self.assertEqual(run(gh(extra=big("# heading\n" * 200_000))).failures, [MSG_SHORT]) + g = gh(extra=big(README * 5_000)) + self.assertEqual(run(g).failures, []) + self.assertIn(("GET", f"{R}/git/blobs/big", None), g.calls) + # A deprecation marker in a big README is seen too. + self.assertEqual(len(run(gh(extra=big("This project is no longer maintained. " + README * 5_000))).failures), 1) + + def test_readme_over_5mb_is_undecided_without_a_download(self): + huge = {f"GET {R}/readme": {"encoding": "none", "content": "", "size": 100_000_000, "sha": "big"}, + f"GET {R}/git/blobs/big": {"encoding": "base64", "sha": "big", "content": b64(README)["content"]}} + g = gh(extra=huge) + for mode in ("submit", "rot"): + with self.assertRaises(Undecided): + run(g, mode) + self.assertFalse([c for c in g.calls if "/git/blobs/" in c[1]]) + # 3 MB is still read. + g = gh(extra={f"GET {R}/readme": {"encoding": "none", "content": "", "size": 3_000_000, "sha": "big"}, + f"GET {R}/git/blobs/big": {"encoding": "base64", "sha": "big", + "content": b64(README * 7_000)["content"]}}) + self.assertEqual(run(g).failures, []) + self.assertIn(("GET", f"{R}/git/blobs/big", None), g.calls) + + def test_readme_encoding_none_without_sha_is_undecided(self): + bad = {"encoding": "none", "content": "", "size": 2_000_000} + with self.assertRaises(Undecided): + run(gh(extra={f"GET {R}/readme": bad})) def test_bad_base64_is_undecided(self): bad = {"encoding": "base64", "content": "abcde"} diff --git a/scripts/tests/test_reports.py b/scripts/tests/test_reports.py index 0f9a514..6142340 100644 --- a/scripts/tests/test_reports.py +++ b/scripts/tests/test_reports.py @@ -1,14 +1,16 @@ +import io import json import os import tempfile +import time import unittest from datetime import datetime, timedelta, timezone from pathlib import Path from unittest import mock from scripts.devstream.entries import load, save -from scripts.devstream.gh import RateLimited -from scripts.reports import handle_report, main +from scripts.devstream.gh import GitHubError, RateLimited +from scripts.reports import ERROR, handle_report, main from scripts.tests.fake_gh import FakeGitHub NOW = datetime(2026, 11, 5, 12, 0, tzinfo=timezone.utc) @@ -55,6 +57,15 @@ def entry(repo="alice/tool", status="live"): } +ROT_RUNS = f"GET {API}/actions/workflows/rot.yml/runs" + + +def rot_runs(*runs_): + """runs_ is (age, title) pairs.""" + return {ROT_RUNS: {"total_count": len(runs_), "workflow_runs": [ + {"created_at": iso(NOW - a), "display_title": t, "status": "completed"} for a, t in runs_]}} + + def search_key(login): return (f"GET /search/issues?q=repo:devstreamhq/devstream+label:report+author:{login}" "+created:>=2026-11-04T12:00:00Z&per_page=100") @@ -79,6 +90,7 @@ def edits(v): return {"repository": {f"i{k}": {"lastEditedAt": iso(NOW) if n in edited else None} for k, n in ((k, v[f"n{k}"]) for k in range(len(v)))}} + routes[ROT_RUNS] = {"total_count": 0, "workflow_runs": []} routes["GRAPHQL UserActivity"] = activity routes["GRAPHQL ReportEdits"] = edits routes.update(extra or {}) @@ -280,6 +292,32 @@ def test_dead_dispatches_rot_with_repo(self): self.assertNotIn("close", kinds(actions)) self.assertEqual(comments(actions)[0], "A health check on `alice/tool` has been queued.") + def test_second_dead_report_within_a_day_does_not_dispatch(self): + dead = report(entry="https://github.com/Alice/Tool", reason="Dead or abandoned") + for title in ("Health check: alice/tool", "Health check: Alice/Tool"): + g = gh(extra=rot_runs((timedelta(hours=23), title))) + actions = run(dead, g) + self.assertNotIn("dispatch_rot", kinds(actions), title) + self.assertEqual(labels(actions), {"needs-review"}) + self.assertIn("in the last day", comments(actions)[0]) + # Checks of other repos, and full runs, don't count. + g = gh(extra=rot_runs((timedelta(hours=2), "Health check: alice/tool-ng"), (timedelta(hours=1), "Health check"))) + self.assertIn("dispatch_rot", kinds(run(dead, g))) + call = next(c for c in g.calls if c[1].startswith(f"{API}/actions/workflows/rot.yml/runs")) + self.assertIn("created=%3E%3D2026-11-04T12%3A00%3A00Z", call[1]) + + def test_fourth_dead_report_within_an_hour_is_parked(self): + dead = report(reason="Dead or abandoned") + three = [(timedelta(minutes=15 * k + 5), f"Health check: x/r{k}") for k in range(3)] + g = gh(extra=rot_runs(*three)) + actions = run(dead, g) + self.assertNotIn("dispatch_rot", kinds(actions)) + self.assertEqual(labels(actions), {"needs-review"}) + self.assertIn("a maintainer", comments(actions)[0]) + # Two in the hour (the third is older) still dispatches. + g = gh(extra=rot_runs(*three[:2], (timedelta(minutes=61), "Health check: x/r2"))) + self.assertIn("dispatch_rot", kinds(run(dead, g))) + def test_superseded_labels_and_validates(self): actions = run(report(reason="Replaced by something better", superseded_by="https://github.com/Bob/New")) self.assertEqual(labels(actions), {"superseded", "needs-review"}) @@ -326,6 +364,66 @@ def write(path, obj): json.dump(obj, f) +class FailedDecideTest(unittest.TestCase): + def decide(self, exc, tmp): + def stuck(*a, **kw): + raise exc + + with mock.patch("scripts.reports.GitHub", lambda token: gh()), \ + mock.patch("scripts.reports.handle_report", stuck), \ + mock.patch("scripts.reports.load", lambda: [entry()]), \ + mock.patch.dict(os.environ, {"GITHUB_TOKEN": "t"}), \ + mock.patch("sys.stderr", io.StringIO()): + ev, out, und = (os.path.join(tmp, f) for f in ("event.json", "result.json", "undecided.json")) + write(ev, {"issue": report()}) + code = main(["decide", "--event", ev, "--out", out, "--undecided", und]) + self.assertFalse(os.path.exists(out)) + return code, und + + def test_failed_decide_is_counted_and_parked_after_three(self): + comments, labels = [], [] + + def post(b): + comments.append({"id": 100 + len(comments), "user": {"login": BOT}, "body": b["body"]}) + + def patch(b): + comments[0]["body"] = b["body"] + + g = FakeGitHub({f"GET {API}/issues/7/comments": lambda _: list(comments), + f"POST {API}/issues/7/comments": post, + f"PATCH {API}/issues/comments/100": patch, + f"POST {API}/issues/7/labels": lambda b: labels.extend(b["labels"])}) + for k in (1, 2, 3): + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(self.decide(GitHubError(502, "bad gateway"), tmp)[0], 3) + with mock.patch("scripts.reports.GitHub", lambda token: g), \ + mock.patch.dict(os.environ, {"GITHUB_TOKEN": "t"}): + self.assertEqual(main(["undecided", "--file", os.path.join(tmp, "undecided.json")]), 0) + self.assertEqual(labels, ["needs-review"]) + self.assertTrue(comments[1]["body"].startswith(ERROR)) + + def test_failed_rot_dispatch_is_counted_too(self): + def boom(_): + raise GitHubError(502, "bad gateway") + + g = FakeGitHub({f"POST {API}/actions/workflows/rot.yml/dispatches": boom}) + with tempfile.TemporaryDirectory() as tmp, \ + mock.patch("scripts.reports.GitHub", lambda token: g), \ + mock.patch.dict(os.environ, {"GITHUB_TOKEN": "t"}), \ + mock.patch("sys.stderr", io.StringIO()): + res, und = os.path.join(tmp, "result.json"), os.path.join(tmp, "undecided.json") + write(res, {"issue": 7, "actions": [{"kind": "dispatch_rot", "value": "alice/tool"}]}) + self.assertEqual(main(["dispatch", "--result", res, "--undecided", und]), 3) + with open(und) as f: + self.assertEqual(json.load(f), {"issue": 7, "error": "GitHub API 502: bad gateway"}) + + def test_rate_limit_still_in_force_spends_no_attempt(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(RateLimited): + self.decide(RateLimited(int(time.time()) + 600), tmp) + self.assertFalse(os.path.exists(os.path.join(tmp, "undecided.json"))) + + class CliTest(unittest.TestCase): def test_dispatch_fetches_issue_and_validates_input(self): for bad in ("7; rm -rf /", "-1", "", "٧", "7\n"): diff --git a/scripts/tests/test_rot.py b/scripts/tests/test_rot.py index da4675f..f0c5c12 100644 --- a/scripts/tests/test_rot.py +++ b/scripts/tests/test_rot.py @@ -1,6 +1,7 @@ import base64 import json import os +import re import tempfile import unittest from datetime import datetime, timedelta, timezone @@ -16,7 +17,7 @@ Undecided, check_repo, ) -from scripts.rot import Action, finish, plan_rot, pr_body, prepare, select +from scripts.rot import Action, finish, open_reviews, plan_rot, pr_body, prepare, select from scripts.tests.fake_gh import FakeGitHub, Redirect NOW = datetime(2026, 11, 5, 6, 0, tzinfo=timezone.utc) @@ -210,6 +211,7 @@ def file(data): f"PUT {API}/contents/discussions.json": None, f"PUT {API}/contents/flags.json": None, f"PUT {API}/contents/renames.json": None, + f"PUT {API}/contents/undecided.json": None, f"GET {API}/git/ref/heads/stats": {"ref": "refs/heads/stats"}, } r.update(extra or {}) @@ -242,7 +244,7 @@ def answer(body): return answer r = {f"GET {API}/git/ref/heads/stats": {"ref": "refs/heads/stats"}} - for path in ("rot.json", "discussions.json", "flags.json", "likes.json", "meta.json", "renames.json"): + for path in ("rot.json", "discussions.json", "flags.json", "likes.json", "meta.json", "renames.json", "undecided.json"): r[f"GET {API}/contents/{path}"] = get(path) r[f"PUT {API}/contents/{path}"] = put(path) r.update(extra or {}) @@ -306,6 +308,52 @@ def check(full, gh, now, mode, **kw): self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("warn", "bob/other")]) self.assertEqual({e["repo"]: e["status"] for e in load(self.path)}, {"alice/tool": "live", "bob/other": "warned"}) + def test_undecided_three_weeks_running_opens_one_review_issue(self): + def check(full, gh, now, mode, **kw): + if full == "alice/tool": + raise Undecided(0, "README of alice/tool could not be decoded") + return ok(full) + + issues = [] + + def open_issue(b): + issues.append({"number": 50 + len(issues), "title": b["title"], "user": {"login": "devstreamhq-bot[bot]"}}) + return issues[-1] + + # A look-alike title from someone else doesn't stand in for the bot's issue. + fake = {"number": 9, "title": "Health check can't decide on alice/tool", "user": {"login": "mallory"}} + r, state = store(extra={f"GET {API}/issues": lambda _: [fake, *issues], f"POST {API}/issues": open_issue}) + g = FakeGitHub(r) + for week in range(5): + plan = prepare(g, NOW + timedelta(days=7 * week), self.path, None, check=check) + self.assertEqual(state["undecided.json"]["alice/tool"]["weeks"], week + 1) + self.assertEqual([r["repo"] for r in plan["review"]], [] if week < 2 else ["alice/tool"]) + # The plan step opens nothing; the review step does, with its own issues-only token. + self.assertFalse([c for c in g.calls if c[:2] == ("POST", f"{API}/issues")][len(issues):]) + open_reviews(g, plan["review"]) + self.assertEqual(len(issues), 0 if week < 2 else 1, week) + posts = [b for m, p, b in g.calls if m == "POST" and p == f"{API}/issues"] + self.assertEqual(posts[0]["labels"], ["needs-review"]) + self.assertIn("`alice/tool`", posts[0]["body"]) + self.assertNotIn("bob/other", state["undecided.json"]) + + # A one-repo run or a second run the same week isn't another week. + prepare(g, NOW + timedelta(days=29), self.path, None, check=check) + prepare(g, NOW + timedelta(days=30), self.path, "alice/tool", check=check) + self.assertEqual(state["undecided.json"]["alice/tool"]["weeks"], 5) + + # Decided again: the count starts over. + prepare(g, NOW + timedelta(days=35), self.path, None, check=lambda full, *a, **kw: ok(full)) + self.assertEqual(state["undecided.json"], {}) + + def test_review_issue_step_has_its_own_issues_only_token(self): + text = (Path(__file__).resolve().parents[2] / ".github" / "workflows" / "rot.yml").read_text() + blocks = text.split("create-github-app-token@") + scopes = [re.findall(r"permission-([\w-]+): (\w+)", b.split("- ")[0]) for b in blocks[1:]] + self.assertIn([("issues", "write")], scopes) + self.assertFalse([sc for sc in scopes if ("issues", "write") in sc and len(sc) > 1]) + self.assertIn("python -m scripts.rot review", text) + def test_skips_hidden_and_superseded(self): save([entry(status="hidden"), entry("bob/other", status="superseded", issue=2)], self.path) seen = [] diff --git a/scripts/tests/test_submit.py b/scripts/tests/test_submit.py index 9b05547..cf03e2b 100644 --- a/scripts/tests/test_submit.py +++ b/scripts/tests/test_submit.py @@ -4,15 +4,20 @@ import subprocess import sys import tempfile +import time import unittest from unittest import mock from datetime import datetime, timedelta, timezone from scripts.devstream.entries import save from scripts.devstream.gh import GitHubError, RateLimited +from scripts.devstream.quality import Undecided +from scripts.devstream.attempts import record_failure from scripts.submit import ( ARCHIVED_DUP, DUPLICATE, NOT_FOUND, RATE_LIMIT, SEED_DUP, comment, decide, main, ) +from scripts.sweep import sweep +from scripts.tests import test_sweep from scripts.validate_data import validate from scripts.tests.fake_gh import FakeGitHub from scripts.tests.test_quality import README, b64, iso, repo @@ -284,6 +289,107 @@ def test_dispatch_skips_handled_or_foreign_issues(self): self.assertFalse(self.wrote) +API = "/repos/devstreamhq/devstream" +BOT = "devstreamhq-bot[bot]" + + +def thread(n, comments, labels): + """Routes for issue n's comments and labels, backed by the lists given.""" + def post(b): + comments.append({"id": 100 + len(comments), "user": {"login": BOT}, "body": b["body"]}) + return comments[-1] + + def patch(cid): + def answer(b): + next(c for c in comments if c["id"] == cid)["body"] = b["body"] + return answer + + routes = {f"GET {API}/issues/{n}/comments": lambda _: list(comments), + f"POST {API}/issues/{n}/comments": post, + f"POST {API}/issues/{n}/labels": lambda b: labels.extend(b["labels"])} + routes.update({f"PATCH {API}/issues/comments/{100 + k}": patch(100 + k) for k in range(10)}) + return routes + + +class UndecidedTest(unittest.TestCase): + def decide(self, exc, tmp, g=None): + def stuck(*a, **kw): + raise exc + + with mock.patch("scripts.submit.GitHub", lambda token: g or gh()), \ + mock.patch("scripts.submit.load", lambda: []), \ + mock.patch("scripts.submit.decide", stuck), \ + mock.patch.dict(os.environ, {"GITHUB_TOKEN": "t"}), \ + mock.patch("sys.stderr", io.StringIO()): + ev, out, und = (os.path.join(tmp, f) for f in ("event.json", "result.json", "undecided.json")) + write(ev, {"issue": issue()}) + code = main(["decide", "--event", ev, "--out", out, "--undecided", und]) + self.assertFalse(os.path.exists(out)) + return code, und + + def test_any_failure_exits_3_and_writes_the_error(self): + for exc in (Undecided(0, "LICENSE of alice/tool could not be decoded"), GitHubError(502, "bad gateway"), + RateLimited(int(time.time()) - 5), TimeoutError("read timed out")): + with tempfile.TemporaryDirectory() as tmp: + code, und = self.decide(exc, tmp) + self.assertEqual(code, 3, exc) + with open(und) as f: + self.assertEqual(json.load(f), {"issue": 7, "error": str(exc)}) + + def test_rate_limit_still_in_force_spends_no_attempt(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(RateLimited): + self.decide(RateLimited(int(time.time()) + 600), tmp) + self.assertFalse(os.path.exists(os.path.join(tmp, "undecided.json"))) + + def test_api_failure_three_times_parks_and_the_sweeper_moves_on(self): + comments, labels = [], [] + g = FakeGitHub(thread(7, comments, labels)) + for k in (1, 2, 3): + with tempfile.TemporaryDirectory() as tmp: + self.assertEqual(self.decide(GitHubError(500, "boom"), tmp)[0], 3) + with mock.patch("scripts.submit.GitHub", lambda token: g), \ + mock.patch.dict(os.environ, {"GITHUB_TOKEN": "t"}): + self.assertEqual(main(["undecided", "--file", os.path.join(tmp, "undecided.json")]), 0) + self.assertEqual(labels, [] if k < 3 else ["needs-review"]) + self.assertEqual(len(comments), 2) + self.assertTrue(comments[1]["body"].startswith(comment({}, "error"))) + self.assertIn("GitHub API 500: boom", comments[1]["body"]) + + s = test_sweep.gh([test_sweep.issue(7, age=timedelta(hours=5), labels=("submission", *labels)), + test_sweep.issue(8, age=timedelta(hours=1))]) + sweep(s, test_sweep.NOW) + self.assertEqual(test_sweep.dispatched(s), ["8"]) + + def test_counter_is_bot_only_and_parks_from_three_on(self): + # The submitter can post a look-alike counter; only the bot's own comment counts. + comments = [{"id": 99, "user": {"login": "carol"}, "body": ""}] + labels = [] + g = FakeGitHub(thread(1, comments, labels)) + err = comment({}, "error") + for k in (1, 2): + self.assertFalse(record_failure(g, 1, "boom", err)) + self.assertEqual(labels, []) + self.assertEqual(len(comments), 2, "one counter comment, edited in place") + self.assertIn(f"", comments[1]["body"]) + self.assertTrue(record_failure(g, 1, "boom `x`", err)) + self.assertEqual(labels, ["needs-review"]) + self.assertEqual(len(comments), 3) + self.assertTrue(comments[2]["body"].startswith(err)) + self.assertIn("boom 'x'", comments[2]["body"]) + # At the limit the counter stops promising a retry. + self.assertNotIn("try again", comments[1]["body"]) + self.assertIn("", comments[1]["body"]) + # A maintainer retries by removing the label; failures 4 and 5 park it again, with no new comment. + for k in (4, 5): + labels.clear() + self.assertTrue(record_failure(g, 1, "boom", err)) + self.assertEqual(labels, ["needs-review"], k) + self.assertEqual(len(comments), 3, k) + self.assertIn(f"", comments[1]["body"]) + self.assertNotIn("try again", comments[1]["body"]) + + class CliTest(unittest.TestCase): def test_apply_duplicate_on_fresh_main_routes_to_reject(self): for status, reason in (("live", DUPLICATE), ("archived", ARCHIVED_DUP)): diff --git a/scripts/tests/test_sweep.py b/scripts/tests/test_sweep.py index 8fe1177..f66cd27 100644 --- a/scripts/tests/test_sweep.py +++ b/scripts/tests/test_sweep.py @@ -201,14 +201,14 @@ def test_submissions_go_first_when_rot_is_not_overdue(self): def test_only_full_runs_count_as_a_health_check(self): # A one-repo re-check doesn't cover the rest of the list. - oks = mixed((timedelta(hours=3), "Health check (one repo)"), (timedelta(days=9), "Health check")) - latest = mixed((timedelta(hours=3), "Health check (one repo)"), (timedelta(days=2), "Health check")) + oks = mixed((timedelta(hours=3), "Health check: alice/tool"), (timedelta(days=9), "Health check")) + latest = mixed((timedelta(hours=3), "Health check: alice/tool"), (timedelta(days=2), "Health check")) g = gh(routes={ROT_OK: oks, ROT_LATEST: latest}) sweep(g, NOW) self.assertEqual(rot_dispatched(g), [{"ref": "main"}]) # Names that only start or end the same way don't count either. - oks = mixed((timedelta(days=1), "Health check (one repo)"), (timedelta(days=1), "Health check "), + oks = mixed((timedelta(days=1), "Health check: alice/tool"), (timedelta(days=1), "Health check "), (timedelta(days=1), "health check")) g = gh(routes={ROT_OK: oks, ROT_LATEST: runs(timedelta(days=2))}) sweep(g, NOW)