diff --git a/scripts/devstream/gh.py b/scripts/devstream/gh.py index 0a2e9bb..f708f84 100644 --- a/scripts/devstream/gh.py +++ b/scripts/devstream/gh.py @@ -38,6 +38,11 @@ def redirect_request(self, *a, **k): return None +def _allowed(e: dict, not_found: bool, forbidden: bool) -> bool: + # A FORBIDDEN with no path is the whole query refused, never one hidden repo. + return (not_found and e.get("type") == "NOT_FOUND") or (forbidden and e.get("type") == "FORBIDDEN" and bool(e.get("path"))) + + class GitHub: def __init__(self, token: str, base: str = "https://api.github.com", sleep=time.sleep): self.token = token @@ -184,8 +189,10 @@ def put(self, path, body): def delete(self, path, body=None): return self._write("DELETE", path, body) - def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> dict: - """allow_not_found returns data with nulls where only NOT_FOUND errors occurred.""" + def graphql(self, query: str, variables: dict, allow_not_found: bool = False, + allow_forbidden: bool = False) -> dict: + """allow_not_found returns data with nulls where only NOT_FOUND errors occurred. + allow_forbidden also lets through FORBIDDEN errors on one alias (an org IP allow list).""" self.last_redirect = None r = self._request("POST", "/graphql", body={"query": query, "variables": variables}, read=bool(READ_QUERY.match(query, LEADING_COMMENTS.match(query).end()))) @@ -195,6 +202,6 @@ def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> errors = out.get("errors") if any(e.get("type") == "RATE_LIMITED" for e in errors or []): raise RateLimited(int(r[0].get("x-ratelimit-reset") or time.time() + 60)) - if errors and not (allow_not_found and all(e.get("type") == "NOT_FOUND" for e in errors)): + if errors and not all(_allowed(e, allow_not_found, allow_forbidden) for e in errors): raise GitHubError(200, json.dumps(out["errors"])[:500]) return out.get("data") diff --git a/scripts/devstream/likes.py b/scripts/devstream/likes.py index 7ed88e8..ea5a016 100644 --- a/scripts/devstream/likes.py +++ b/scripts/devstream/likes.py @@ -82,7 +82,7 @@ def fetch_reactors(gh, numbers: list[int]) -> dict[int, list[dict] | None]: def fetch_meta(gh, repos: list[str]) -> dict[str, dict]: - """Repo card data keyed as given. A deleted repo is left out; rot handles it.""" + """Repo card data keyed as given. A deleted or IP-allow-listed repo is left out; rot handles the deleted.""" out = {} for i in range(0, len(repos), META_BATCH): chunk = repos[i:i + META_BATCH] @@ -91,7 +91,8 @@ def fetch_meta(gh, repos: list[str]) -> dict[str, dict]: variables = {} for j, r in enumerate(chunk): variables[f"o{j}"], variables[f"n{j}"] = r.split("/", 1) - data = gh.graphql(f"query RepoMeta({params}) {{ {body} }}", variables, allow_not_found=True) + data = gh.graphql(f"query RepoMeta({params}) {{ {body} }}", variables, allow_not_found=True, + allow_forbidden=True) for j, r in enumerate(chunk): m = data.get(f"r{j}") if m is None: diff --git a/scripts/rot.py b/scripts/rot.py index bd55bda..bd37c73 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -17,7 +17,7 @@ from scripts.devstream.discussions import REMOVED_PREFIX from scripts.devstream.entries import DEFAULT_PATH, find, load, normalize_repo, save, sort_key -from scripts.devstream.gh import GitHub +from scripts.devstream.gh import GitHub, GitHubError from scripts.devstream.license_detect import extra_licenses_for from scripts.devstream.quality import ( MSG_GONE, @@ -203,7 +203,7 @@ def select(entries, raw): def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: - """Check, plan and write the new entries.json locally. Any GitHubError aborts before a write.""" + """Check, plan and write the new entries.json locally. Any other GitHubError aborts before a write.""" entries = load(path) targets = [e for e in select(entries, repo) if e["status"] in CHECKED] ensure_branch(gh) @@ -214,7 +214,10 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: for e in targets: try: results[e["repo"]] = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"])) - except Undecided as exc: + except GitHubError as exc: + # An owner's IP allow list hides the repo from our token; that says nothing about the repo. + if not isinstance(exc, Undecided) and not (exc.status == 403 and "IP allow list" in exc.message): + raise logging.warning("skipping %s: %s", e["repo"], exc) undecided[normalize_repo(e["repo"])] = str(exc) review = [] diff --git a/scripts/sync_likes.py b/scripts/sync_likes.py index aae021d..ad6dd74 100644 --- a/scripts/sync_likes.py +++ b/scripts/sync_likes.py @@ -227,7 +227,10 @@ def run(gh, entries, now: datetime, excluded: set[str]) -> None: if _rename_in_flight(gh, now, entries, keys, mapping, prev_likes or {}): return - meta = fetch_meta(gh, keys) + prev_meta = read_json(gh, "meta.json")[0] + # A repo GitHub hides from us this hour keeps its last card rather than losing stars and avatar. + listed = set(keys) + meta = {k: v for k, v in (prev_meta or {}).items() if k in listed} | fetch_meta(gh, keys) reactors = fetch_reactors(gh, sorted({mapping[k] for k in keys if k in mapping})) prev_flags = read_json(gh, "flags.json")[0] flags = copy.deepcopy(prev_flags or {}) @@ -283,7 +286,6 @@ def trusted(login): "flagged": ranked[k]["status"] == "flagged" or k in bursts or _under_review(flags, k), } - prev_meta = read_json(gh, "meta.json")[0] snap_path = f"snapshots/{now:%Y-%m-%d}.json" flags_write = ("flags.json", _merge_flags(flags, prev_flags or {}), "chore(stats): record Like reviews") diff --git a/scripts/tests/fake_gh.py b/scripts/tests/fake_gh.py index be6f2a1..193f4e7 100644 --- a/scripts/tests/fake_gh.py +++ b/scripts/tests/fake_gh.py @@ -1,7 +1,7 @@ import re import urllib.parse -from scripts.devstream.gh import GitHubError +from scripts.devstream.gh import GitHubError, _allowed class Redirect: @@ -121,7 +121,8 @@ def put(self, path, body): def delete(self, path, body=None): return self._write("DELETE", path, body) - def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> dict: + def graphql(self, query: str, variables: dict, allow_not_found: bool = False, + allow_forbidden: bool = False) -> dict: self.last_redirect = None m = re.search(r"\b(?:query|mutation)\s+(\w+)", query) self.calls.append(("GRAPHQL", m.group(1) if m else "", variables)) @@ -130,7 +131,7 @@ def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> v = self.routes[key] v = v(variables) if callable(v) else v if isinstance(v, GraphQLErrors): - if not (allow_not_found and all(e.get("type") == "NOT_FOUND" for e in v.errors)): + if not all(_allowed(e, allow_not_found, allow_forbidden) for e in v.errors): raise GitHubError(200, str(v.errors)) return v.data return v diff --git a/scripts/tests/test_gh.py b/scripts/tests/test_gh.py index 9376eed..9af51b8 100644 --- a/scripts/tests/test_gh.py +++ b/scripts/tests/test_gh.py @@ -123,6 +123,8 @@ def do_POST(self): errs = [{"type": "NOT_FOUND", "path": ["r0"]}] if body["variables"]["nf"] == 2: errs.append({"type": "FORBIDDEN"}) + if body["variables"]["nf"] == 3: + errs.append({"type": "FORBIDDEN", "path": ["r1"]}) self._send(200, {"data": {"r0": None}, "errors": errs}) elif body["variables"].get("bad"): self._send(200, {"errors": [{"message": "x"}]}) @@ -209,6 +211,10 @@ def test_graphql_allow_not_found(self): self.gh.graphql("query Q { x }", {"nf": 1}) with self.assertRaises(GitHubError): self.gh.graphql("query Q { x }", {"nf": 2}, allow_not_found=True) + with self.assertRaises(GitHubError): # pathless FORBIDDEN is the whole query, not one alias + self.gh.graphql("query Q { x }", {"nf": 2}, allow_not_found=True, allow_forbidden=True) + self.assertEqual(self.gh.graphql("query Q { x }", {"nf": 3}, allow_not_found=True, allow_forbidden=True), + {"r0": None}) def test_write_success(self): for m in ("post", "patch", "put"): @@ -414,6 +420,8 @@ def test_graphql_errors_route(self): f.graphql("query Q { x }", {}) with self.assertRaises(GitHubError): f.graphql("query B { x }", {}, allow_not_found=True) + with self.assertRaises(GitHubError): + f.graphql("query B { x }", {}, allow_not_found=True, allow_forbidden=True) def test_status_routes(self): f = FakeGitHub({"GET /b": Status(204), "GET /x": {"a": 1}, "GET /e": Status(500)}) diff --git a/scripts/tests/test_likes.py b/scripts/tests/test_likes.py index 65ffd09..2153667 100644 --- a/scripts/tests/test_likes.py +++ b/scripts/tests/test_likes.py @@ -101,9 +101,9 @@ def _meta(self, v): while f"o{i}" in v: key = f"{v[f'o{i}']}/{v[f'n{i}']}" stars = self.stars.get(key, 0) - if stars is None: - errors.append({"type": "NOT_FOUND", "path": [f"r{i}"]}) - out[f"r{i}"] = None if stars is None else { + if stars is None or stars == "forbidden": + errors.append({"type": "NOT_FOUND" if stars is None else "FORBIDDEN", "path": [f"r{i}"]}) + out[f"r{i}"] = None if stars is None or stars == "forbidden" else { "nameWithOwner": key, "stargazerCount": stars, "primaryLanguage": {"name": "Go", "color": "#00ADD8"}, "pushedAt": "2026-11-01T00:00:00Z", "owner": {"avatarUrl": "https://a/x.png"}} @@ -725,6 +725,17 @@ def test_meta_shape_and_skips_deleted_repo(self): test_meta_skips_deleted_repo = test_meta_shape_and_skips_deleted_repo + def test_meta_forbidden_repo_keeps_last_card(self): + # An org IP allow list hides one repo from the App token; the rest of the run goes on. + old = {"stars": 9, "language": "Go", "language_color": "#00ADD8", + "pushed_at": "2026-10-01T00:00:00Z", "avatar_url": "https://a/old.png"} + w = World([entry("o/a"), entry("o/ip")], {"o/a": [node("x")], "o/ip": [node("y")]}, + stars={"o/a": 3, "o/ip": "forbidden"}, stats={"meta.json": {"o/ip": old}}) + w.run() + meta = w.written("meta.json") + self.assertEqual((meta["o/a"]["stars"], meta["o/ip"]), (3, old)) + self.assertEqual(w.written("likes.json")["o/ip"]["likes"], 1) + def test_meta_batched_per_50(self): repos = {f"o/r{i:03}": [] for i in range(51)} w = World([entry(r) for r in repos], repos) diff --git a/scripts/tests/test_rot.py b/scripts/tests/test_rot.py index f0c5c12..274100d 100644 --- a/scripts/tests/test_rot.py +++ b/scripts/tests/test_rot.py @@ -308,6 +308,23 @@ def check(full, gh, now, mode, **kw): self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("warn", "bob/other")]) self.assertEqual({e["repo"]: e["status"] for e in load(self.path)}, {"alice/tool": "live", "bob/other": "warned"}) + def test_ip_allow_listed_repo_is_undecided_not_an_abort(self): + def check(full, gh, now, mode, **kw): + if full == "alice/tool": + raise GitHubError(403, "the `alice` organization has an IP allow list enabled") + return warn(full) + + plan = self.run_prepare(FakeGitHub(routes()), check) + self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("warn", "bob/other")]) + self.assertEqual({e["repo"]: e["status"] for e in load(self.path)}, {"alice/tool": "live", "bob/other": "warned"}) + + def test_other_403_still_aborts(self): + def check(full, gh, now, mode, **kw): + raise GitHubError(403, "Resource not accessible by integration") + + with self.assertRaises(GitHubError): + self.run_prepare(FakeGitHub(routes()), check) + def test_undecided_three_weeks_running_opens_one_review_issue(self): def check(full, gh, now, mode, **kw): if full == "alice/tool":