Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions scripts/devstream/gh.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ def redirect_request(self, *a, **k):
return None


def _allowed(e: dict, not_found: bool, forbidden: bool) -> bool:
# A FORBIDDEN with no path is the whole query refused, never one hidden repo.
return (not_found and e.get("type") == "NOT_FOUND") or (forbidden and e.get("type") == "FORBIDDEN" and bool(e.get("path")))


class GitHub:
def __init__(self, token: str, base: str = "https://api.github.com", sleep=time.sleep):
self.token = token
Expand Down Expand Up @@ -184,8 +189,10 @@ def put(self, path, body):
def delete(self, path, body=None):
return self._write("DELETE", path, body)

def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> dict:
"""allow_not_found returns data with nulls where only NOT_FOUND errors occurred."""
def graphql(self, query: str, variables: dict, allow_not_found: bool = False,
allow_forbidden: bool = False) -> dict:
"""allow_not_found returns data with nulls where only NOT_FOUND errors occurred.
allow_forbidden also lets through FORBIDDEN errors on one alias (an org IP allow list)."""
self.last_redirect = None
r = self._request("POST", "/graphql", body={"query": query, "variables": variables},
read=bool(READ_QUERY.match(query, LEADING_COMMENTS.match(query).end())))
Expand All @@ -195,6 +202,6 @@ def graphql(self, query: str, variables: dict, allow_not_found: bool = False) ->
errors = out.get("errors")
if any(e.get("type") == "RATE_LIMITED" for e in errors or []):
raise RateLimited(int(r[0].get("x-ratelimit-reset") or time.time() + 60))
if errors and not (allow_not_found and all(e.get("type") == "NOT_FOUND" for e in errors)):
if errors and not all(_allowed(e, allow_not_found, allow_forbidden) for e in errors):
raise GitHubError(200, json.dumps(out["errors"])[:500])
return out.get("data")
5 changes: 3 additions & 2 deletions scripts/devstream/likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def fetch_reactors(gh, numbers: list[int]) -> dict[int, list[dict] | None]:


def fetch_meta(gh, repos: list[str]) -> dict[str, dict]:
"""Repo card data keyed as given. A deleted repo is left out; rot handles it."""
"""Repo card data keyed as given. A deleted or IP-allow-listed repo is left out; rot handles the deleted."""
out = {}
for i in range(0, len(repos), META_BATCH):
chunk = repos[i:i + META_BATCH]
Expand All @@ -91,7 +91,8 @@ def fetch_meta(gh, repos: list[str]) -> dict[str, dict]:
variables = {}
for j, r in enumerate(chunk):
variables[f"o{j}"], variables[f"n{j}"] = r.split("/", 1)
data = gh.graphql(f"query RepoMeta({params}) {{ {body} }}", variables, allow_not_found=True)
data = gh.graphql(f"query RepoMeta({params}) {{ {body} }}", variables, allow_not_found=True,
allow_forbidden=True)
for j, r in enumerate(chunk):
m = data.get(f"r{j}")
if m is None:
Expand Down
9 changes: 6 additions & 3 deletions scripts/rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

from scripts.devstream.discussions import REMOVED_PREFIX
from scripts.devstream.entries import DEFAULT_PATH, find, load, normalize_repo, save, sort_key
from scripts.devstream.gh import GitHub
from scripts.devstream.gh import GitHub, GitHubError
from scripts.devstream.license_detect import extra_licenses_for
from scripts.devstream.quality import (
MSG_GONE,
Expand Down Expand Up @@ -203,7 +203,7 @@ def select(entries, raw):


def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict:
"""Check, plan and write the new entries.json locally. Any GitHubError aborts before a write."""
"""Check, plan and write the new entries.json locally. Any other GitHubError aborts before a write."""
entries = load(path)
targets = [e for e in select(entries, repo) if e["status"] in CHECKED]
ensure_branch(gh)
Expand All @@ -214,7 +214,10 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict:
for e in targets:
try:
results[e["repo"]] = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"]))
except Undecided as exc:
except GitHubError as exc:
# An owner's IP allow list hides the repo from our token; that says nothing about the repo.
if not isinstance(exc, Undecided) and not (exc.status == 403 and "IP allow list" in exc.message):
raise
logging.warning("skipping %s: %s", e["repo"], exc)
undecided[normalize_repo(e["repo"])] = str(exc)
review = []
Expand Down
6 changes: 4 additions & 2 deletions scripts/sync_likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,10 @@ def run(gh, entries, now: datetime, excluded: set[str]) -> None:
if _rename_in_flight(gh, now, entries, keys, mapping, prev_likes or {}):
return

meta = fetch_meta(gh, keys)
prev_meta = read_json(gh, "meta.json")[0]
# A repo GitHub hides from us this hour keeps its last card rather than losing stars and avatar.
listed = set(keys)
meta = {k: v for k, v in (prev_meta or {}).items() if k in listed} | fetch_meta(gh, keys)
reactors = fetch_reactors(gh, sorted({mapping[k] for k in keys if k in mapping}))
prev_flags = read_json(gh, "flags.json")[0]
flags = copy.deepcopy(prev_flags or {})
Expand Down Expand Up @@ -283,7 +286,6 @@ def trusted(login):
"flagged": ranked[k]["status"] == "flagged" or k in bursts or _under_review(flags, k),
}

prev_meta = read_json(gh, "meta.json")[0]
snap_path = f"snapshots/{now:%Y-%m-%d}.json"
flags_write = ("flags.json", _merge_flags(flags, prev_flags or {}), "chore(stats): record Like reviews")

Expand Down
7 changes: 4 additions & 3 deletions scripts/tests/fake_gh.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import re
import urllib.parse

from scripts.devstream.gh import GitHubError
from scripts.devstream.gh import GitHubError, _allowed


class Redirect:
Expand Down Expand Up @@ -121,7 +121,8 @@ def put(self, path, body):
def delete(self, path, body=None):
return self._write("DELETE", path, body)

def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> dict:
def graphql(self, query: str, variables: dict, allow_not_found: bool = False,
allow_forbidden: bool = False) -> dict:
self.last_redirect = None
m = re.search(r"\b(?:query|mutation)\s+(\w+)", query)
self.calls.append(("GRAPHQL", m.group(1) if m else "", variables))
Expand All @@ -130,7 +131,7 @@ def graphql(self, query: str, variables: dict, allow_not_found: bool = False) ->
v = self.routes[key]
v = v(variables) if callable(v) else v
if isinstance(v, GraphQLErrors):
if not (allow_not_found and all(e.get("type") == "NOT_FOUND" for e in v.errors)):
if not all(_allowed(e, allow_not_found, allow_forbidden) for e in v.errors):
raise GitHubError(200, str(v.errors))
return v.data
return v
Expand Down
8 changes: 8 additions & 0 deletions scripts/tests/test_gh.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ def do_POST(self):
errs = [{"type": "NOT_FOUND", "path": ["r0"]}]
if body["variables"]["nf"] == 2:
errs.append({"type": "FORBIDDEN"})
if body["variables"]["nf"] == 3:
errs.append({"type": "FORBIDDEN", "path": ["r1"]})
self._send(200, {"data": {"r0": None}, "errors": errs})
elif body["variables"].get("bad"):
self._send(200, {"errors": [{"message": "x"}]})
Expand Down Expand Up @@ -209,6 +211,10 @@ def test_graphql_allow_not_found(self):
self.gh.graphql("query Q { x }", {"nf": 1})
with self.assertRaises(GitHubError):
self.gh.graphql("query Q { x }", {"nf": 2}, allow_not_found=True)
with self.assertRaises(GitHubError): # pathless FORBIDDEN is the whole query, not one alias
self.gh.graphql("query Q { x }", {"nf": 2}, allow_not_found=True, allow_forbidden=True)
self.assertEqual(self.gh.graphql("query Q { x }", {"nf": 3}, allow_not_found=True, allow_forbidden=True),
{"r0": None})

def test_write_success(self):
for m in ("post", "patch", "put"):
Expand Down Expand Up @@ -414,6 +420,8 @@ def test_graphql_errors_route(self):
f.graphql("query Q { x }", {})
with self.assertRaises(GitHubError):
f.graphql("query B { x }", {}, allow_not_found=True)
with self.assertRaises(GitHubError):
f.graphql("query B { x }", {}, allow_not_found=True, allow_forbidden=True)

def test_status_routes(self):
f = FakeGitHub({"GET /b": Status(204), "GET /x": {"a": 1}, "GET /e": Status(500)})
Expand Down
17 changes: 14 additions & 3 deletions scripts/tests/test_likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,9 @@ def _meta(self, v):
while f"o{i}" in v:
key = f"{v[f'o{i}']}/{v[f'n{i}']}"
stars = self.stars.get(key, 0)
if stars is None:
errors.append({"type": "NOT_FOUND", "path": [f"r{i}"]})
out[f"r{i}"] = None if stars is None else {
if stars is None or stars == "forbidden":
errors.append({"type": "NOT_FOUND" if stars is None else "FORBIDDEN", "path": [f"r{i}"]})
out[f"r{i}"] = None if stars is None or stars == "forbidden" else {
"nameWithOwner": key, "stargazerCount": stars,
"primaryLanguage": {"name": "Go", "color": "#00ADD8"},
"pushedAt": "2026-11-01T00:00:00Z", "owner": {"avatarUrl": "https://a/x.png"}}
Expand Down Expand Up @@ -725,6 +725,17 @@ def test_meta_shape_and_skips_deleted_repo(self):

test_meta_skips_deleted_repo = test_meta_shape_and_skips_deleted_repo

def test_meta_forbidden_repo_keeps_last_card(self):
# An org IP allow list hides one repo from the App token; the rest of the run goes on.
old = {"stars": 9, "language": "Go", "language_color": "#00ADD8",
"pushed_at": "2026-10-01T00:00:00Z", "avatar_url": "https://a/old.png"}
w = World([entry("o/a"), entry("o/ip")], {"o/a": [node("x")], "o/ip": [node("y")]},
stars={"o/a": 3, "o/ip": "forbidden"}, stats={"meta.json": {"o/ip": old}})
w.run()
meta = w.written("meta.json")
self.assertEqual((meta["o/a"]["stars"], meta["o/ip"]), (3, old))
self.assertEqual(w.written("likes.json")["o/ip"]["likes"], 1)

def test_meta_batched_per_50(self):
repos = {f"o/r{i:03}": [] for i in range(51)}
w = World([entry(r) for r in repos], repos)
Expand Down
17 changes: 17 additions & 0 deletions scripts/tests/test_rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,23 @@ def check(full, gh, now, mode, **kw):
self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("warn", "bob/other")])
self.assertEqual({e["repo"]: e["status"] for e in load(self.path)}, {"alice/tool": "live", "bob/other": "warned"})

def test_ip_allow_listed_repo_is_undecided_not_an_abort(self):
def check(full, gh, now, mode, **kw):
if full == "alice/tool":
raise GitHubError(403, "the `alice` organization has an IP allow list enabled")
return warn(full)

plan = self.run_prepare(FakeGitHub(routes()), check)
self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("warn", "bob/other")])
self.assertEqual({e["repo"]: e["status"] for e in load(self.path)}, {"alice/tool": "live", "bob/other": "warned"})

def test_other_403_still_aborts(self):
def check(full, gh, now, mode, **kw):
raise GitHubError(403, "Resource not accessible by integration")

with self.assertRaises(GitHubError):
self.run_prepare(FakeGitHub(routes()), check)

def test_undecided_three_weeks_running_opens_one_review_issue(self):
def check(full, gh, now, mode, **kw):
if full == "alice/tool":
Expand Down
Loading