diff --git a/scripts/rot.py b/scripts/rot.py index 6d98aa3..a59ab55 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -12,13 +12,14 @@ import os import re import sys +import time from dataclasses import asdict, dataclass from datetime import datetime, timedelta, timezone from zoneinfo import ZoneInfo from scripts.devstream.discussions import REMOVED_PREFIX from scripts.devstream.entries import DEFAULT_PATH, find, load, normalize_repo, save, sort_key -from scripts.devstream.gh import GitHub, GitHubError +from scripts.devstream.gh import GitHub, GitHubError, RateLimited from scripts.devstream.license_detect import extra_licenses_for from scripts.devstream.quality import ( MSG_GONE, @@ -34,6 +35,8 @@ from scripts.validate_data import validate CHECKED = {"live", "warned", "archived"} +# GitHub's primary limit resets hourly; a longer wait than that is something else. +MAX_WAIT = 3600 GRACE = timedelta(days=30) # A second sighting this much later confirms a removal, so a blip never deletes an entry. CONFIRM = timedelta(days=6) @@ -234,26 +237,34 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: undecided = {} replaced = [] for e in targets: - try: - r = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"])) - if e.get("repo_id") is None: - logging.warning("%s has no repo_id, so a replaced repo can't be caught", e["repo"]) - elif r.exists and r.repo_id is not None and r.repo_id != e["repo_id"]: - got = r.repo_id - # The owner may have moved (an account rename) and someone took the old name: follow the ID. - moved = get_repo(gh, f"/repositories/{e['repo_id']}") - r = moved and check(moved["full_name"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"])) - if not (r and r.exists and r.repo_id == e["repo_id"]): - # Someone else's repo at the listed name inherits nothing: it reads as gone, and a person decides. - replaced.append({"repo": e["repo"], "kind": "replaced", "want": e["repo_id"], "got": got}) - r = RepoResult(False, e["repo"], [MSG_REPLACED], "remove") - results[e["repo"]] = r - except GitHubError as exc: - # An owner's IP allow list hides the repo from our token; that says nothing about the repo. - if not isinstance(exc, Undecided) and not (exc.status == 403 and "IP allow list" in exc.message): - raise - logging.warning("skipping %s: %s", e["repo"], exc) - undecided[normalize_repo(e["repo"])] = str(exc) + for waited in (False, True): + try: + r = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"])) + if e.get("repo_id") is None: + logging.warning("%s has no repo_id, so a replaced repo can't be caught", e["repo"]) + elif r.exists and r.repo_id is not None and r.repo_id != e["repo_id"]: + got = r.repo_id + # The owner may have moved (an account rename) and someone took the old name: follow the ID. + moved = get_repo(gh, f"/repositories/{e['repo_id']}") + r = moved and check(moved["full_name"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"])) + if not (r and r.exists and r.repo_id == e["repo_id"]): + # Someone else's repo at the listed name inherits nothing: it reads as gone, and a person decides. + replaced.append({"repo": e["repo"], "kind": "replaced", "want": e["repo_id"], "got": got}) + r = RepoResult(False, e["repo"], [MSG_REPLACED], "remove") + results[e["repo"]] = r + except GitHubError as exc: + # An owner's IP allow list hides the repo from our token; that says nothing about the repo. + if not isinstance(exc, Undecided) and not (exc.status == 403 and "IP allow list" in exc.message): + raise + logging.warning("skipping %s: %s", e["repo"], exc) + undecided[normalize_repo(e["repo"])] = str(exc) + except RateLimited as exc: + # Past ~700 entries the check outruns the hourly budget: wait for the reset, then retry this entry once. + if waited or exc.reset_at - time.time() > MAX_WAIT: + raise + gh.sleep(max(0, exc.reset_at - time.time())) + continue + break review = [] if repo is None: try: diff --git a/scripts/tests/fake_gh.py b/scripts/tests/fake_gh.py index 74e3400..1c9b5d9 100644 --- a/scripts/tests/fake_gh.py +++ b/scripts/tests/fake_gh.py @@ -55,13 +55,15 @@ class FakeGitHub: .last_redirect is reset at the start of every call. - .calls records (method, path, body) for every request. + .calls records (method, path, body) for every request; .slept, every sleep. """ def __init__(self, routes: dict[str, object]): self.routes = routes self.calls: list[tuple[str, str, object]] = [] self.last_redirect: str | None = None + self.slept: list[float] = [] + self.sleep = self.slept.append def _find(self, method, path, params, body, write=False, hops=0): full = path diff --git a/scripts/tests/test_rot.py b/scripts/tests/test_rot.py index 8e61adc..6c3675a 100644 --- a/scripts/tests/test_rot.py +++ b/scripts/tests/test_rot.py @@ -3,6 +3,7 @@ import os import re import tempfile +import time import unittest from datetime import datetime, timedelta, timezone from pathlib import Path @@ -298,6 +299,29 @@ def check(full, gh, now, mode, **kw): self.assertEqual(Path(self.path).read_text(), before) self.assertEqual(writes(g), []) + def test_rate_limit_is_waited_out_then_the_entry_retried(self): + hits = [] + + def check(full, gh, now, mode, **kw): + if full == "bob/other" and not hits: + hits.append(full) + raise RateLimited(0) + return warn(full) + + g = FakeGitHub(routes()) + plan = self.run_prepare(g, check) + self.assertEqual(g.slept, [0]) + self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("warn", "alice/tool"), ("warn", "bob/other")]) + + def test_rate_limit_over_an_hour_away_aborts_without_waiting(self): + def check(full, gh, now, mode, **kw): + raise RateLimited(int(time.time()) + 2 * 3600) + + g = FakeGitHub(routes()) + with self.assertRaises(RateLimited): + self.run_prepare(g, check) + self.assertEqual(g.slept, []) + def test_undecided_repo_is_skipped_not_failed(self): def check(full, gh, now, mode, **kw): if full == "alice/tool":