Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 32 additions & 21 deletions scripts/rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,14 @@
import os
import re
import sys
import time
from dataclasses import asdict, dataclass
from datetime import datetime, timedelta, timezone
from zoneinfo import ZoneInfo

from scripts.devstream.discussions import REMOVED_PREFIX
from scripts.devstream.entries import DEFAULT_PATH, find, load, normalize_repo, save, sort_key
from scripts.devstream.gh import GitHub, GitHubError
from scripts.devstream.gh import GitHub, GitHubError, RateLimited
from scripts.devstream.license_detect import extra_licenses_for
from scripts.devstream.quality import (
MSG_GONE,
Expand All @@ -34,6 +35,8 @@
from scripts.validate_data import validate

CHECKED = {"live", "warned", "archived"}
# GitHub's primary limit resets hourly; a longer wait than that is something else.
MAX_WAIT = 3600
GRACE = timedelta(days=30)
# A second sighting this much later confirms a removal, so a blip never deletes an entry.
CONFIRM = timedelta(days=6)
Expand Down Expand Up @@ -234,26 +237,34 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict:
undecided = {}
replaced = []
for e in targets:
try:
r = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"]))
if e.get("repo_id") is None:
logging.warning("%s has no repo_id, so a replaced repo can't be caught", e["repo"])
elif r.exists and r.repo_id is not None and r.repo_id != e["repo_id"]:
got = r.repo_id
# The owner may have moved (an account rename) and someone took the old name: follow the ID.
moved = get_repo(gh, f"/repositories/{e['repo_id']}")
r = moved and check(moved["full_name"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"]))
if not (r and r.exists and r.repo_id == e["repo_id"]):
# Someone else's repo at the listed name inherits nothing: it reads as gone, and a person decides.
replaced.append({"repo": e["repo"], "kind": "replaced", "want": e["repo_id"], "got": got})
r = RepoResult(False, e["repo"], [MSG_REPLACED], "remove")
results[e["repo"]] = r
except GitHubError as exc:
# An owner's IP allow list hides the repo from our token; that says nothing about the repo.
if not isinstance(exc, Undecided) and not (exc.status == 403 and "IP allow list" in exc.message):
raise
logging.warning("skipping %s: %s", e["repo"], exc)
undecided[normalize_repo(e["repo"])] = str(exc)
for waited in (False, True):
try:
r = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"]))
if e.get("repo_id") is None:
logging.warning("%s has no repo_id, so a replaced repo can't be caught", e["repo"])
elif r.exists and r.repo_id is not None and r.repo_id != e["repo_id"]:
got = r.repo_id
# The owner may have moved (an account rename) and someone took the old name: follow the ID.
moved = get_repo(gh, f"/repositories/{e['repo_id']}")
r = moved and check(moved["full_name"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"]))
if not (r and r.exists and r.repo_id == e["repo_id"]):
# Someone else's repo at the listed name inherits nothing: it reads as gone, and a person decides.
replaced.append({"repo": e["repo"], "kind": "replaced", "want": e["repo_id"], "got": got})
r = RepoResult(False, e["repo"], [MSG_REPLACED], "remove")
results[e["repo"]] = r
except GitHubError as exc:
# An owner's IP allow list hides the repo from our token; that says nothing about the repo.
if not isinstance(exc, Undecided) and not (exc.status == 403 and "IP allow list" in exc.message):
raise
logging.warning("skipping %s: %s", e["repo"], exc)
undecided[normalize_repo(e["repo"])] = str(exc)
except RateLimited as exc:
# Past ~700 entries the check outruns the hourly budget: wait for the reset, then retry this entry once.
if waited or exc.reset_at - time.time() > MAX_WAIT:
raise
gh.sleep(max(0, exc.reset_at - time.time()))
continue
break
review = []
if repo is None:
try:
Expand Down
4 changes: 3 additions & 1 deletion scripts/tests/fake_gh.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,15 @@ class FakeGitHub:

.last_redirect is reset at the start of every call.

.calls records (method, path, body) for every request.
.calls records (method, path, body) for every request; .slept, every sleep.
"""

def __init__(self, routes: dict[str, object]):
self.routes = routes
self.calls: list[tuple[str, str, object]] = []
self.last_redirect: str | None = None
self.slept: list[float] = []
self.sleep = self.slept.append

def _find(self, method, path, params, body, write=False, hops=0):
full = path
Expand Down
24 changes: 24 additions & 0 deletions scripts/tests/test_rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import os
import re
import tempfile
import time
import unittest
from datetime import datetime, timedelta, timezone
from pathlib import Path
Expand Down Expand Up @@ -298,6 +299,29 @@ def check(full, gh, now, mode, **kw):
self.assertEqual(Path(self.path).read_text(), before)
self.assertEqual(writes(g), [])

def test_rate_limit_is_waited_out_then_the_entry_retried(self):
hits = []

def check(full, gh, now, mode, **kw):
if full == "bob/other" and not hits:
hits.append(full)
raise RateLimited(0)
return warn(full)

g = FakeGitHub(routes())
plan = self.run_prepare(g, check)
self.assertEqual(g.slept, [0])
self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("warn", "alice/tool"), ("warn", "bob/other")])

def test_rate_limit_over_an_hour_away_aborts_without_waiting(self):
def check(full, gh, now, mode, **kw):
raise RateLimited(int(time.time()) + 2 * 3600)

g = FakeGitHub(routes())
with self.assertRaises(RateLimited):
self.run_prepare(g, check)
self.assertEqual(g.slept, [])

def test_undecided_repo_is_skipped_not_failed(self):
def check(full, gh, now, mode, **kw):
if full == "alice/tool":
Expand Down
Loading