diff --git a/scripts/devstream/discussions.py b/scripts/devstream/discussions.py index 024e072..2d3210f 100644 --- a/scripts/devstream/discussions.py +++ b/scripts/devstream/discussions.py @@ -31,7 +31,7 @@ def list_discussions(gh, category_id: str) -> dict[str, int]: """Every thread already in the category, keyed by normalised repo.""" - out, cursor = {}, None + out, cursor, seen = {}, None, set() while True: page = gh.graphql(_LIST, {"cursor": cursor, "cat": category_id})["repository"]["discussions"] for n in page["nodes"]: @@ -45,6 +45,10 @@ def list_discussions(gh, category_id: str) -> dict[str, int]: if not page["pageInfo"]["hasNextPage"]: return out cursor = page["pageInfo"]["endCursor"] + # A cursor seen before would loop until the job times out. + if not cursor or cursor in seen: + raise GitHubError(200, "discussions cursor repeated") + seen.add(cursor) def likes_category(gh) -> tuple[str, str]: diff --git a/scripts/devstream/gh.py b/scripts/devstream/gh.py index 022b2ff..b9b736b 100644 --- a/scripts/devstream/gh.py +++ b/scripts/devstream/gh.py @@ -1,3 +1,4 @@ +import http.client import json import random import re @@ -6,6 +7,8 @@ import urllib.error import urllib.parse import urllib.request +from datetime import timezone +from email.utils import parsedate_to_datetime from typing import Iterator MAX_SLEEP = 600 @@ -13,7 +16,8 @@ TIMEOUT = 30 ATTEMPTS = 3 TRANSIENT = (502, 503, 504) -NETWORK = (urllib.error.URLError, TimeoutError, ConnectionError) +# OSError covers URLError, timeouts, resets and ssl; http.client raises its own on a cut-off response. +NETWORK = (OSError, http.client.HTTPException) # Only a document that opens as a query is a read; anything else (a fragment first, say) may be a mutation. LEADING_COMMENTS = re.compile(r"(\s*#[^\n]*(\n|$))*") READ_QUERY = re.compile(r"\s*(query\b|\{)") @@ -40,6 +44,8 @@ def redirect_request(self, *a, **k): def _allowed(e: dict, not_found: bool, forbidden: bool) -> bool: # A FORBIDDEN with no path is the whole query refused, never one hidden repo. + if not isinstance(e, dict): + return False return (not_found and e.get("type") == "NOT_FOUND") or (forbidden and e.get("type") == "FORBIDDEN" and bool(e.get("path"))) @@ -79,7 +85,14 @@ def _rate_limit_reset(status, headers) -> int | None: if headers.get("x-ratelimit-remaining") != "0" and retry is None: return None if retry is not None: - return int(time.time()) + int(retry) + if retry.strip().isdigit(): + return int(time.time()) + int(retry) + # Retry-After may also be an HTTP date; anything unreadable waits a minute. + try: + at = parsedate_to_datetime(retry) + except (TypeError, ValueError): + return int(time.time()) + 60 + return int((at if at.tzinfo else at.replace(tzinfo=timezone.utc)).timestamp()) return int(headers.get("x-ratelimit-reset") or time.time() + 60) @staticmethod @@ -109,7 +122,8 @@ def _request(self, method, path, params=None, body=None, read=None): except NETWORK as e: fails += 1 if fails >= ATTEMPTS or not self._retryable(method, read, exc=e): - raise + # As a GitHubError, the callers' per-item error handling applies to it too. + raise GitHubError(0, f"network error: {e}") from e self._backoff(fails) continue if self._retryable(method, read, status) and fails + 1 < ATTEMPTS: @@ -128,7 +142,9 @@ def _request(self, method, path, params=None, body=None, read=None): hops += 1 if hops > MAX_HOPS: raise GitHubError(status, "too many redirects") - loc = urllib.parse.urlsplit(headers["Location"]) + if not headers.get("Location"): + raise GitHubError(status, "redirect without a Location") + loc = urllib.parse.urlsplit(urllib.parse.urljoin(self.base + path, headers["Location"])) path = loc.path + (f"?{loc.query}" if loc.query else "") self.last_redirect = path continue @@ -164,7 +180,11 @@ def paginate(self, path, params=None, must_exist=False) -> Iterator[dict]: raise GitHubError(404, f"page not found: {path}") first = False headers, page = r - yield from page["items"] if isinstance(page, dict) and "items" in page else page + items = page["items"] if isinstance(page, dict) and "items" in page else page + if not isinstance(items, list): + # A dict here would yield its keys as if they were items. + raise GitHubError(200, f"not a list page: {path}") + yield from items m = re.search(r'<([^>]+)>\s*;\s*rel="next"', headers.get("Link") or "") # Rebuilt against base so the token never leaves the API host. nxt = urllib.parse.urlsplit(m.group(1)) if m else None @@ -201,7 +221,7 @@ def graphql(self, query: str, variables: dict, allow_not_found: bool = False, raise GitHubError(404, "graphql endpoint not found") out = r[1] or {} errors = out.get("errors") - if any(e.get("type") == "RATE_LIMITED" for e in errors or []): + if any(isinstance(e, dict) and e.get("type") == "RATE_LIMITED" for e in errors or []): raise RateLimited(int(r[0].get("x-ratelimit-reset") or time.time() + 60)) if errors and not all(_allowed(e, allow_not_found, allow_forbidden) for e in errors): raise GitHubError(200, json.dumps(out["errors"])[:500]) diff --git a/scripts/on_merge.py b/scripts/on_merge.py index 04b4d81..40a691c 100644 --- a/scripts/on_merge.py +++ b/scripts/on_merge.py @@ -12,6 +12,19 @@ VISIBLE = {"live", "warned", "flagged"} +def _follow(mapping: dict, pending: dict, listed: set[str]) -> dict: + """Finish a rename whose key move failed in rot's finish, so the entry keeps its thread and its Likes. + + pending holds only renames.json records rot marked pending; a completed move is never replayed. + Overrides a stored new key: that is the dropped entry's thread after a rename collision. + An old name listed again is someone else's entry and keeps its own key. + """ + for old, r in sorted(pending.items(), key=lambda kv: kv[1]["at"]): + if old in mapping and old not in listed: + mapping[normalize_repo(r["to"])] = mapping.pop(old) + return mapping + + def run(gh, entries, sleep=time.sleep): ensure_branch(gh) cur = read_json(gh, "discussions.json")[0] @@ -22,6 +35,11 @@ def run(gh, entries, sleep=time.sleep): # setdefault: a stored mapping always wins, so a thread retitled onto a listed name can't take it over. for k, v in list_discussions(gh, category_id).items(): mapping.setdefault(k, v) + # After adopting, so a thread still titled with the old name can't put the old key back. + pending = {k: r for k, r in (read_json(gh, "renames.json")[0] or {}).items() + if isinstance(r, dict) and r.get("pending")} + listed = {normalize_repo(e["repo"]) for e in entries} + _follow(mapping, pending, listed) todo = {} for e in entries: if e["status"] in VISIBLE and normalize_repo(e["repo"]) not in mapping: @@ -40,15 +58,22 @@ def run(gh, entries, sleep=time.sleep): added = {k: v for k, v in mapping.items() if before.get(k) != v} def merge(stored): - # Re-applied onto a fresh read after a 409; a stored mapping still wins. - return {**added, **{normalize_repo(k): v for k, v in stored.items()}} - if mapping != before: - try: + # Re-applied onto a fresh read after a 409; a stored mapping still wins, once renames are followed. + return {**added, **_follow({normalize_repo(k): v for k, v in stored.items()}, pending, listed)} + + def done(stored): + # Replayed once: a later relist and unlist of the old name must never pull its thread back. + return {k: {f: x for f, x in v.items() if f != "pending"} if pending.get(k) == v else v + for k, v in stored.items()} + try: + if mapping != before: update_json(gh, "discussions.json", merge, "chore(stats): map entry discussions") - except Exception: - if err is None: - raise - logging.exception("could not write discussions.json") + if pending: + update_json(gh, "renames.json", done, "chore(stats): finish renamed repos") + except Exception: + if err is None: + raise + logging.exception("could not write discussions.json or renames.json") def main(): diff --git a/scripts/rot.py b/scripts/rot.py index dfe06f7..cd423c2 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -342,31 +342,33 @@ def record(data): # sync_likes replays these over snapshots taken before the rename; past its 7-day baseline they're unused. kept = {k: v for k, v in data.items() if isinstance(v, dict) and now - _parse(v["at"]) < RENAME_KEEP} # renames.json keeps one record per old name; a repeat rename replaces it, fine on a weekly schedule. - return {**kept, **{old: {"to": new, "at": _iso(now)} for old, new in moves.items()}} + # pending: the key move failed, so on_merge replays this record once; a completed move is never replayed. + mark = {"pending": True} if "discussions.json" in failed else {} + return {**kept, **{old: {"to": new, "at": _iso(now), **mark} for old, new in moves.items()}} numbers = dict(read_json(gh, "discussions.json")[0] or {}) # before the move, so dropped entries find their thread - failed = 0 - moved = True + failed = set() for path, mutate, message in ( ("rot.json", lambda _: plan["rot_state"], "chore(stats): record failing repos"), ("discussions.json", rekey, "chore(stats): follow renamed repos"), + # After the key move, so record knows whether on_merge has to finish it. + ("renames.json", record, "chore(stats): record renamed repos"), ("flags.json", rekey, "chore(stats): follow renamed repos"), # The site build reads these two by key, so they move now rather than at the next hourly sync. ("likes.json", rekey, "chore(stats): follow renamed repos"), ("meta.json", rekey, "chore(stats): follow renamed repos"), - ("renames.json", record, "chore(stats): record renamed repos"), ): try: update_json(gh, path, mutate, message) except Exception: logging.exception("could not write %s", path) - failed += 1 - moved = moved and path != "discussions.json" - # A retitle without its key move would leave the thread named for a key it isn't stored under. - posted, notified = _post(gh, actions, numbers, retitle=moved) - failed += posted + _mark_notified(gh, notified, plan["rot_state"]) - if failed: - raise SystemExit(f"{failed} Discussion or stats update(s) failed") + failed.add(path) + # A title must match a key that is stored, or will be once on_merge replays the pending record. + retitle = {"rename": not {"discussions.json", "renames.json"} <= failed, "drop": "discussions.json" not in failed} + posted, notified = _post(gh, actions, numbers, retitle) + total = len(failed) + posted + _mark_notified(gh, notified, plan["rot_state"]) + if total: + raise SystemExit(f"{total} Discussion or stats update(s) failed") def _mark_notified(gh, keys: set[str], planned: dict) -> int: @@ -392,7 +394,7 @@ def mark(data): return 0 -def _post(gh, actions, numbers: dict, retitle: bool = True) -> tuple[int, set[str]]: +def _post(gh, actions, numbers: dict, retitle: dict | None = None) -> tuple[int, set[str]]: """Comment on, or retitle, each action's Discussion. numbers maps stats key to Discussion number. Returns (failures, rot keys whose notice is done). @@ -408,7 +410,7 @@ def _post(gh, actions, numbers: dict, retitle: bool = True) -> tuple[int, set[st if d is not None: if a.kind != "rename": gh.graphql(_COMMENT, {"discussionId": d["id"], "body": a.message}) - if a.kind in ("rename", "drop") and retitle: + if a.kind in ("rename", "drop") and (retitle or {}).get(a.kind, True): # "Removed: " can never normalise to a listed repo, so on_merge won't adopt the thread. title = a.repo if a.kind == "rename" else f"{REMOVED_PREFIX}{a.repo}" gh.graphql(_RETITLE, {"discussionId": d["id"], "title": title}) diff --git a/scripts/sync_likes.py b/scripts/sync_likes.py index 2eb5116..e7c4c9f 100644 --- a/scripts/sync_likes.py +++ b/scripts/sync_likes.py @@ -31,6 +31,9 @@ WEEK = timedelta(days=7) ISSUE_CAP = 5 LABELS = ["report", "needs-review"] +BOT = "devstreamhq-bot[bot]" +AGGREGATE_TITLE = re.compile(r"Like burst: [0-9]+ entr(y|ies)") +AGGREGATE_LINE = re.compile(r"^- (\S+)$", re.M) # How long a reviewed (closed) issue holds off a new one for the same entry. COOLDOWN = {"burst": DAY, "drip": WEEK} SNAPSHOTS_KEPT = 30 @@ -66,8 +69,10 @@ def _review(gh, flags: dict, now: datetime) -> None: try: issue = gh.get(f"{API}/issues/{n}") except GitHubError as e: - if e.status != 410: # 410: deleted - raise + if e.status not in (404, 410): # deleted + # Its record stays pending and is looked up again next hour; the rest still sync. + logging.warning("could not look up issue #%s: %s", n, e) + continue issue = None if issue is None or issue.get("state") == "closed": done.add(n) @@ -145,10 +150,20 @@ def _baseline(gh, now: datetime, renames: dict) -> dict | None: return data +def _open_reports(gh) -> dict[str, dict]: + """The bot's open Like issues by title. Only the bot can author these, so a title can't be forged here.""" + return {i["title"]: i for i in gh.paginate(f"{API}/issues", {"state": "open", "labels": LABELS[0], + "per_page": 100}) + if (i.get("user") or {}).get("login") == BOT and "pull_request" not in i} + + def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetime) -> None: """todo is (kind, key, repo, body). Records are set right after each post, so a failure keeps them.""" + # A failed flags.json write loses the records of issues already open; adopt those rather than open them again. + opened = _open_reports(gh) for kind, key, repo, body in todo[:ISSUE_CAP]: - n = gh.post(f"{API}/issues", {"title": f"Like {kind}: {repo}", "body": body, "labels": LABELS})["number"] + title = f"Like {kind}: {repo}" + n = (opened.get(title) or {}).get("number") or gh.post(f"{API}/issues", {"title": title, "body": body, "labels": LABELS})["number"] flags.setdefault(key, {})[kind] = _record(kind, n, now) overflow = [(key, kind) for kind, key, _, _ in todo[ISSUE_CAP:]] if not overflow: @@ -156,6 +171,12 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim # A flood of decoys must not become a flood of issues, nor let the target slip its flag. agg = next((r["issue"] for recs in flags.values() for r in recs.values() if r.get("aggregate") and r["reviewed_at"] is None), None) + kept = [] + if agg is None: + adopted = next((i for t, i in opened.items() if AGGREGATE_TITLE.fullmatch(t)), None) + if adopted is not None: + # Its records were lost, so the entries it already lists come from its body. + agg, kept = adopted["number"], AGGREGATE_LINE.findall(adopted.get("body") or "") if agg is None: agg = gh.post(f"{API}/issues", {**_aggregate(overflow), "labels": LABELS})["number"] for key, kind in overflow: @@ -165,7 +186,20 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim flags.setdefault(key, {})[kind] = _record(kind, agg, now, aggregate=True) listed = [(k, kind) for k, recs in flags.items() for kind, r in recs.items() if r["issue"] == agg and r["reviewed_at"] is None] - gh.patch(f"{API}/issues/{agg}", _aggregate(listed)) + listed += [(k, None) for k in kept if k not in {key for key, _ in listed}] + try: + state = (gh.patch(f"{API}/issues/{agg}", _aggregate(listed)) or {}).get("state") + except GitHubError as e: + if e.status not in (404, 410): + # The records already point at it, so the next overflow brings the body up to date. + logging.warning("could not update aggregate issue #%s: %s", agg, e) + return + state = "closed" + if state == "closed": + # Closed or deleted since _review: its review is over, so these entries need an issue of their own. + agg = gh.post(f"{API}/issues", {**_aggregate(overflow), "labels": LABELS})["number"] + for key, kind in overflow: + flags[key][kind] = _record(kind, agg, now, aggregate=True) def _merge_flags(flags: dict, before: dict): @@ -184,7 +218,7 @@ def mutate(cur): def _rename_in_flight(gh, now, entries, keys, mapping, prev_likes) -> bool: - """True while a rename has merged but rot's finish hasn't moved discussions.json yet. + """True while a rename has merged but discussions.json hasn't moved yet (rot's finish, or on_merge within the hour). The tell: a listed key has no thread, and a key that was ranked last time has a thread but is no longer listed. An entry removed for good drops out of likes.json, so it can't hold the board. diff --git a/scripts/tests/fake_gh.py b/scripts/tests/fake_gh.py index db9c4ab..74e3400 100644 --- a/scripts/tests/fake_gh.py +++ b/scripts/tests/fake_gh.py @@ -1,7 +1,7 @@ import re import urllib.parse -from scripts.devstream.gh import GitHubError, _allowed +from scripts.devstream.gh import MAX_HOPS, GitHubError, _allowed class Redirect: @@ -50,7 +50,8 @@ class FakeGitHub: Only .get returns None on a missing route; post, patch, put and delete raise GitHubError(404). A route whose value is None answers a write like a 204. paginate yields nothing for a missing route (the real client does - the same for a 404 on the first page, unless must_exist; the fake has a single page). + the same for a 404 on the first page, unless must_exist; the fake has a single page), + and raises for a routed page that isn't a list. .last_redirect is reset at the start of every call. @@ -62,7 +63,7 @@ def __init__(self, routes: dict[str, object]): self.calls: list[tuple[str, str, object]] = [] self.last_redirect: str | None = None - def _find(self, method, path, params, body, write=False): + def _find(self, method, path, params, body, write=False, hops=0): full = path if params: full += ("&" if "?" in path else "?") + urllib.parse.urlencode(params) @@ -72,8 +73,11 @@ def _find(self, method, path, params, body, write=False): if key in self.routes: v = self.routes[key] if isinstance(v, Redirect): + # Same limit as the real client, so a cycle fails instead of recursing forever. + if hops >= MAX_HOPS: + raise GitHubError(301, "too many redirects") self.last_redirect = v.path - return self._find(method, v.path, None, body, write) + return self._find(method, v.path, None, body, write, hops + 1) return v(body) if callable(v) else v if write: raise GitHubError(404, f"no route: {method} {path}") @@ -101,13 +105,20 @@ def status(self, path) -> int: def paginate(self, path, params=None, must_exist=False): self.last_redirect = None page = self._find("GET", path, params, None) + # A route answering None is an empty body, which the real client refuses; no route is a 404. + full = path + (("&" if "?" in path else "?") + urllib.parse.urlencode(params) if params else "") + routed = page is not None or any(f"GET {p}" in self.routes for p in (full, path.split("?")[0])) if isinstance(page, Status) and self._ok(page): - page = None - if page is None and must_exist: - raise GitHubError(404, f"page not found: {path}") + page, routed = None, False + if page is None and not routed: + if must_exist: + raise GitHubError(404, f"page not found: {path}") + return if isinstance(page, dict) and "items" in page: page = page["items"] - yield from page or [] + if not isinstance(page, list): + raise GitHubError(200, f"not a list page: {path}") + yield from page def _write(self, method, path, body): self.last_redirect = None diff --git a/scripts/tests/test_discussions.py b/scripts/tests/test_discussions.py index 162cb72..85702ad 100644 --- a/scripts/tests/test_discussions.py +++ b/scripts/tests/test_discussions.py @@ -1,6 +1,6 @@ import unittest -from scripts.devstream.discussions import ensure_discussion, likes_category +from scripts.devstream.discussions import ensure_discussion, likes_category, list_discussions from scripts.devstream.gh import GitHubError from scripts.tests.fake_gh import FakeGitHub @@ -56,5 +56,34 @@ def test_missing_raises(self): self.assertIn("Discussion category 'likes' is missing", c.exception.message) +class Listing(unittest.TestCase): + def pages(self, cursors): + """One thread per page; cursors[i] is the endCursor page i hands back, None on the last.""" + def page(v): + i = 0 if v["cursor"] is None else cursors.index(v["cursor"]) + 1 + return {"repository": {"discussions": { + "nodes": [{"number": i + 1, "title": f"o/r{i}", "author": {"login": "devstreamhq-bot[bot]"}}], + "pageInfo": {"hasNextPage": cursors[i] is not None, "endCursor": cursors[i]}}}} + return FakeGitHub({"GRAPHQL LikesDiscussions": page}) + + def test_follows_every_page(self): + g = self.pages(["c1", "c2", None]) + self.assertEqual(list_discussions(g, "C"), {"o/r0": 1, "o/r1": 2, "o/r2": 3}) + self.assertEqual([v["cursor"] for _, _, v in g.calls], [None, "c1", "c2"]) + + def test_repeated_cursor_raises(self): + g = FakeGitHub({"GRAPHQL LikesDiscussions": {"repository": {"discussions": { + "nodes": [], "pageInfo": {"hasNextPage": True, "endCursor": "stuck"}}}}}) + with self.assertRaises(GitHubError): + list_discussions(g, "C") + self.assertEqual(len(g.calls), 2) + + def test_missing_cursor_raises(self): + g = FakeGitHub({"GRAPHQL LikesDiscussions": {"repository": {"discussions": { + "nodes": [], "pageInfo": {"hasNextPage": True, "endCursor": None}}}}}) + with self.assertRaises(GitHubError): + list_discussions(g, "C") + + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_gh.py b/scripts/tests/test_gh.py index 70dd4e5..695ad3e 100644 --- a/scripts/tests/test_gh.py +++ b/scripts/tests/test_gh.py @@ -1,9 +1,12 @@ +import http.client import json import socket +import ssl import threading import urllib.error import time import unittest +from email.utils import formatdate from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from scripts.devstream.gh import GitHub, GitHubError, RateLimited @@ -78,6 +81,9 @@ def do_GET(self): elif p == "/xlink": nxt = f"; rel=\"next\"" self._send(200, [{"n": 1}], [("Link", nxt)]) + elif p == "/n1": + nxt = f"; rel=\"next\"" + self._send(200, [{"n": 1}], [("Link", nxt)]) elif p == "/twice": self._send(429, {}, [("retry-after", "1")]) elif p == "/q1": @@ -189,6 +195,12 @@ def test_500_is_error(self): def test_paginate_follows_link_header(self): self.assertEqual([i["n"] for i in self.gh.paginate("/p1")], [1, 2, 3]) + def test_paginate_non_list_page_raises(self): + # Iterating a dict would yield its keys as if they were items. + for path in ("/repositories/123", "/n1"): + with self.assertRaises(GitHubError): + list(self.gh.paginate(path)) + def test_paginate_search_items(self): self.assertEqual(list(self.gh.paginate("/search")), [{"n": 9}]) @@ -351,7 +363,7 @@ def test_graphql_query_timeout_retried(self): def test_graphql_mutation_502_not_retried(self): for err in (502, TimeoutError()): gh, sent, _ = self.gh(err, 200) - with self.assertRaises((GitHubError, TimeoutError)): + with self.assertRaises(GitHubError): gh.graphql("mutation M { x }", {}) self.assertEqual(len(sent), 1) @@ -384,9 +396,19 @@ def test_get_network_error_retried(self): self.assertEqual(gh.get("/a"), {"ok": True}) self.assertEqual(len(sent), 2) + def test_ssl_and_http_client_errors_are_network_errors(self): + for err in (ssl.SSLError("bad record"), http.client.IncompleteRead(b"")): + gh, sent, _ = self.gh(err, 200) + self.assertEqual(gh.get("/a"), {"ok": True}) + self.assertEqual(len(sent), 2) + gh, sent, _ = self.gh(err, 200) + with self.assertRaises(GitHubError) as c: + gh.post("/a", {}) + self.assertEqual((c.exception.status, len(sent)), (0, 1)) + def test_three_network_errors_raise(self): gh, sent, _ = self.gh(TimeoutError(), TimeoutError(), TimeoutError(), 200) - with self.assertRaises(TimeoutError): + with self.assertRaises(GitHubError): gh.get("/a") self.assertEqual(len(sent), 3) @@ -399,7 +421,7 @@ def test_write_retried_only_when_never_sent(self): self.assertEqual(len(sent), 2) for err in (TimeoutError(), ConnectionResetError(), urllib.error.URLError("reset")): gh, sent, _ = self.gh(err, 200) - with self.assertRaises(OSError): + with self.assertRaises(GitHubError): gh.patch("/a", {}) self.assertEqual(len(sent), 1) @@ -412,7 +434,70 @@ def test_rate_limit_rules_unchanged(self): self.assertEqual(len(sent), 1) +class OddAnswers(unittest.TestCase): + """_send is scripted with (status, headers, body) answers.""" + + def gh(self, *script): + slept = [] + gh = GitHub("tok", "http://x", sleep=slept.append) + sent = [] + + def send(method, url, body): + sent.append(url) + return script[len(sent) - 1] + gh._send = send + return gh, sent, slept + + def test_redirect_without_location_raises(self): + gh, sent, _ = self.gh((301, {}, b"")) + with self.assertRaises(GitHubError) as c: + gh.get("/repos/a/b") + self.assertEqual(c.exception.status, 301) + + def test_relative_redirect_followed(self): + for loc, want in (("/repositories/9", "/repositories/9"), ("9", "/repositories/9")): + gh, sent, _ = self.gh((301, {"Location": loc}, b""), (200, {}, b'{"ok": 1}')) + self.assertEqual(gh.get("/repositories/old"), {"ok": 1}) + self.assertEqual((sent[1], gh.last_redirect), ("http://x" + want, want)) + + def test_retry_after_http_date(self): + gh, sent, slept = self.gh((429, {"retry-after": formatdate(time.time() + 30, usegmt=True)}, b"{}"), + (200, {}, b'{"ok": 1}')) + self.assertEqual(gh.get("/a"), {"ok": 1}) + self.assertAlmostEqual(slept[0], 30, delta=3) + gh, _, _ = self.gh((429, {"retry-after": formatdate(LONG, usegmt=True)}, b"{}")) + with self.assertRaises(RateLimited) as c: + gh.get("/a") + self.assertEqual(c.exception.reset_at, LONG) + + def test_retry_after_garbage_waits_a_minute(self): + gh, _, slept = self.gh((429, {"retry-after": "soon"}, b"{}"), (200, {}, b'{"ok": 1}')) + self.assertEqual(gh.get("/a"), {"ok": 1}) + self.assertAlmostEqual(slept[0], 60, delta=3) + + def test_graphql_error_that_is_not_a_dict(self): + for errors in (["boom"], [None], [{"type": "NOT_FOUND", "path": ["r0"]}, "boom"]): + gh, _, _ = self.gh((200, {}, json.dumps({"data": {"r0": None}, "errors": errors}).encode())) + with self.assertRaises(GitHubError): + gh.graphql("query Q { x }", {}, allow_not_found=True) + + def test_network_error_after_retries_is_a_github_error(self): + gh = GitHub("tok", "http://x", sleep=lambda s: None) + + def send(*a): + raise urllib.error.URLError("down") + gh._send = send + with self.assertRaises(GitHubError) as c: + gh.get("/a") + self.assertIsInstance(c.exception.__cause__, urllib.error.URLError) + + class FakeTest(unittest.TestCase): + def test_redirect_cycle_raises(self): + f = FakeGitHub({"GET /a": Redirect("/b"), "GET /b": Redirect("/a")}) + with self.assertRaises(GitHubError): + f.get("/a") + def test_same_public_methods(self): pub = lambda c: {n for n in dir(c) if not n.startswith("_") and callable(getattr(c, n))} self.assertEqual(pub(GitHub), pub(FakeGitHub)) @@ -451,6 +536,11 @@ def test_last_redirect_reset(self): f.get("/c") self.assertIsNone(f.last_redirect) + def test_paginate_non_list_page_raises(self): + for page in (None, {"a": 1}): + with self.assertRaises(GitHubError): + list(FakeGitHub({"GET /x": page}).paginate("/x")) + def test_paginate_missing_first_page_empty(self): self.assertEqual(list(FakeGitHub({}).paginate("/x")), []) with self.assertRaises(GitHubError): diff --git a/scripts/tests/test_likes.py b/scripts/tests/test_likes.py index f261993..7a3cbad 100644 --- a/scripts/tests/test_likes.py +++ b/scripts/tests/test_likes.py @@ -58,6 +58,7 @@ def __init__(self, entries, discussions, users=None, stats=None, stars=None, iss "GRAPHQL DiscussionReactors": self._reactors, "GRAPHQL RepoMeta": self._meta, f"POST {API}/issues": self._open_issue, + f"GET {API}/issues": self._list_issues, } for login, created in users.items(): routes[f"GET /users/{login}"] = { @@ -115,10 +116,15 @@ def _meta(self, v): def _open_issue(self, body): n = self.next_issue self.next_issue += 1 - self.routes[f"GET {API}/issues/{n}"] = {"number": n, "state": "open", "title": body["title"]} + self.routes[f"GET {API}/issues/{n}"] = {"number": n, "state": "open", "title": body["title"], + "user": {"login": "devstreamhq-bot[bot]"}} self.routes[f"PATCH {API}/issues/{n}"] = {} return {"number": n} + def _list_issues(self, _): + return [{"user": {"login": "mallory"}, **v} for k, v in self.routes.items() + if k.startswith(f"GET {API}/issues/") and isinstance(v, dict) and v.get("state") == "open"] + def close(self, n): self.routes[f"GET {API}/issues/{n}"]["state"] = "closed" @@ -496,6 +502,112 @@ def boom(_): self.assertEqual(w.issues_opened(), []) self.assertTrue(w.written("likes.json")["o/a"]["flagged"]) + def test_failed_flags_write_after_issues_opens_no_duplicates(self): + repos = {f"o/r{j}": burst_nodes(10, 10) for j in range(6)} + w = World([entry(r) for r in repos], repos) + + def boom(_): + raise GitHubError(502, "bad gateway") + w.routes[f"PUT {API}/contents/flags.json"] = boom + with self.assertRaises(GitHubError): + w.run() + self.assertEqual(len(w.issues_opened()), 6) + w.persist() + w.routes.pop(f"GET {API}/contents/flags.json") # that PUT never landed + w.routes[f"PUT {API}/contents/flags.json"] = {} + w.run(now=NOW + timedelta(minutes=5)) + # The lost records are rebuilt from the bot's open issues, not opened again. + self.assertEqual(w.issues_opened(), []) + flags = w.written("flags.json") + self.assertEqual(sorted(r["burst"]["issue"] for r in flags.values()), [100, 101, 102, 103, 104, 105]) + + def test_aggregate_patch_failure_keeps_the_run(self): + first = {f"o/r{j}": burst_nodes(10, 10) for j in range(6)} + later = {f"o/s{j}": [] for j in range(6)} + w = World([entry(r) for r in {**first, **later}], {**first, **later}) + w.run() + agg = w.next_issue - 1 + w.persist() + + def boom(_): + raise GitHubError(502, "bad gateway") + w.routes[f"PATCH {API}/issues/{agg}"] = boom + for r in later: + w.set_nodes(r, burst_nodes(10, 10, reacted=iso(NOW + timedelta(minutes=30)))) + with self.assertLogs(level="WARNING"): + w.run(now=NOW + timedelta(hours=1)) + self.assertEqual(w.written("flags.json")["o/s5"]["burst"]["issue"], agg) + self.assertIn("o/s5", w.written("likes.json")) + + def test_aggregate_closed_mid_edit_gets_a_new_issue(self): + first = {f"o/r{j}": burst_nodes(10, 10) for j in range(6)} + later = {f"o/s{j}": [] for j in range(6)} + for gone in ({"state": "closed"}, None): + w = World([entry(r) for r in {**first, **later}], {**first, **later}) + w.run() + agg = w.next_issue - 1 + w.persist() + if gone is None: + del w.routes[f"PATCH {API}/issues/{agg}"] # deleted: the fake answers 404 + else: + w.routes[f"PATCH {API}/issues/{agg}"] = gone + for r in later: + w.set_nodes(r, burst_nodes(10, 10, reacted=iso(NOW + timedelta(minutes=30)))) + w.run(now=NOW + timedelta(hours=1)) + self.assertEqual(titles(w)[-1], "Like burst: 1 entry") + self.assertEqual(w.written("flags.json")["o/s5"]["burst"]["issue"], w.next_issue - 1) + self.assertNotEqual(w.next_issue - 1, agg) + + def test_review_lookup_404_counts_as_reviewed(self): + w = World([entry("o/a")], {"o/a": [node("x")]}, stats=self.flags()) + + def gone(_): + raise GitHubError(404, "Not Found") + w.routes[f"GET {API}/issues/7"] = gone + w.run() + self.assertEqual(w.written("flags.json")["o/a"]["burst"]["reviewed_at"], iso(NOW)) + + def test_adopted_aggregate_keeps_the_entries_it_listed(self): + first = {f"o/r{j}": burst_nodes(10, 10) for j in range(6)} + later = {f"o/s{j}": [] for j in range(6)} + w = World([entry(r) for r in {**first, **later}], {**first, **later}) + + def boom(_): + raise GitHubError(502, "bad gateway") + w.routes[f"PUT {API}/contents/flags.json"] = boom + with self.assertRaises(GitHubError): + w.run() + agg = w.next_issue - 1 + w.routes[f"GET {API}/issues/{agg}"]["body"] = w.issues_opened()[-1]["body"] + w.persist() + w.routes.pop(f"GET {API}/contents/flags.json") # that PUT never landed + w.routes[f"PUT {API}/contents/flags.json"] = {} + w.set_nodes("o/r5", []) + for r in later: + w.set_nodes(r, burst_nodes(10, 10, reacted=iso(NOW + timedelta(minutes=30)))) + w.run(now=NOW + timedelta(hours=1)) + [(_, path, body)] = w.patches() + self.assertEqual(path, f"{API}/issues/{agg}") + self.assertEqual(body["title"], "Like burst: 7 entries") + self.assertIn("- o/r5", body["body"]) + self.assertIn("- o/s5", body["body"]) + + def test_one_failed_review_lookup_skips_only_that_issue(self): + stats = {"flags.json": {r: {"burst": {"kind": "burst", "issue": n, "opened_at": iso(NOW), + "reviewed_at": None}} for r, n in (("o/a", 7), ("o/b", 8))}} + w = World([entry("o/a"), entry("o/b")], {"o/a": [], "o/b": []}, stats=stats, + issues=[{"number": 8, "state": "closed"}]) + + def boom(_): + raise GitHubError(502, "bad gateway") + w.routes[f"GET {API}/issues/7"] = boom + with self.assertLogs(level="WARNING"): + w.run() + flags = w.written("flags.json") + self.assertIsNone(flags["o/a"]["burst"]["reviewed_at"]) + self.assertEqual(flags["o/b"]["burst"]["reviewed_at"], iso(NOW)) + self.assertTrue(w.written("likes.json")["o/a"]["flagged"]) + def test_failed_issue_post_keeps_flags_for_opened_ones(self): repos = {f"o/r{j}": burst_nodes(10, 10) for j in range(2)} w = World([entry(r) for r in repos], repos) diff --git a/scripts/tests/test_rot.py b/scripts/tests/test_rot.py index 4dce5f3..709a68f 100644 --- a/scripts/tests/test_rot.py +++ b/scripts/tests/test_rot.py @@ -674,51 +674,102 @@ def test_failed_rot_write_never_starts_the_clock_on_an_old_record(self): new, actions, _ = plan_rot([entry()], {"alice/tool": gone()}, NOW + timedelta(days=1), state["rot.json"]) self.assertEqual((new, kinds(actions)), ([entry()], [("notice", "alice/tool")])) - def test_no_retitle_when_the_key_move_fails(self): - titles = {5: "alice/tool", 6: "bob/kept"} - r, state = store({"discussions.json": {"alice/tool": 5, "bob/kept": 6}}, threads(titles), + def on_merge_routes(self, titles): + def list_threads(_): + nodes = [{"number": n, "title": t, "author": {"login": "devstreamhq-bot[bot]"}} + for n, t in sorted(titles.items())] + return {"repository": {"discussions": { + "nodes": nodes, "pageInfo": {"hasNextPage": False, "endCursor": None}}}} + return { + **threads(titles), + "GRAPHQL LikesCategory": {"repository": {"id": "R", "discussionCategories": { + "nodes": [{"id": "C", "slug": "likes"}]}}}, + "GRAPHQL LikesDiscussions": list_threads, + "GRAPHQL CreateEntryDiscussion": {"createDiscussion": {"discussion": {"number": 99}}}, + } + + def test_failed_key_move_is_finished_by_the_next_on_merge(self): + from scripts.on_merge import run as on_merge + + titles = {5: "alice/tool", 6: "bob/other"} + r, state = store({"discussions.json": {"alice/tool": 5, "bob/other": 6}}, self.on_merge_routes(titles), conflicts={"discussions.json": 3}) g = FakeGitHub(r) with self.assertRaises(SystemExit): finish(g, self.plan([Action("rename", "alice/tool-ng", "", "alice/tool"), - Action("drop", "bob/kept", "Merged", "bob/kept"), - Action("warn", "alice/tool-ng", "Heads up", "alice/tool")])) - self.assertEqual(self.gql(g, "UpdateEntryDiscussion"), []) - self.assertEqual(titles, {5: "alice/tool", 6: "bob/kept"}) - self.assertEqual(sorted(c["body"] for c in self.gql(g, "CommentOnEntry")), ["Heads up", "Merged"]) - - def test_collision_keeps_the_older_entrys_thread_through_on_merge(self): - from scripts.devstream.entries import normalize_repo + Action("warn", "alice/tool-ng", "Heads up", "alice/tool")]), now=NOW) + self.assertIn("alice/tool", state["discussions.json"]) + self.assertEqual(state["renames.json"], + {"alice/tool": {"to": "alice/tool-ng", "at": iso(NOW), "pending": True}}) + self.assertEqual(titles[5], "alice/tool-ng") + self.assertEqual([c["body"] for c in self.gql(g, "CommentOnEntry")], ["Heads up"]) + + listed = [entry("alice/tool-ng"), entry("bob/other", issue=2)] + on_merge(g, listed, sleep=lambda s: None) + self.assertEqual(state["discussions.json"], {"alice/tool-ng": 5, "bob/other": 6, "other/writer": 1}) + self.assertEqual(state["renames.json"], {"alice/tool": {"to": "alice/tool-ng", "at": iso(NOW)}}) + self.assertEqual(self.gql(g, "CreateEntryDiscussion"), []) + g.calls.clear() + on_merge(g, listed, sleep=lambda s: None) + self.assertEqual([c for c in g.calls if c[0] == "PUT"], []) + + def test_completed_move_is_never_replayed_after_a_relist_and_unlist(self): from scripts.on_merge import run as on_merge - from scripts.validate_data import REPO_RE - older = entry("alice/tool", submitted_at="2026-01-01T00:00:00Z", issue=1) - newer = entry("bob/kept", submitted_at="2026-03-01T00:00:00Z", issue=2) - new, actions, rot = plan_rot([older, newer], {"alice/tool": ok("bob/kept")}, NOW, {}) - titles = {5: "alice/tool", 6: "bob/kept"} - listing = {"repository": {"discussions": {"nodes": [], "pageInfo": {"hasNextPage": False, "endCursor": None}}}} + titles = {5: "alice/tool"} + r, state = store({"discussions.json": {"alice/tool": 5}}, self.on_merge_routes(titles), + conflicts={"discussions.json": 3}) + g = FakeGitHub(r) + with self.assertRaises(SystemExit): + finish(g, self.plan([Action("rename", "alice/tool-ng", "", "alice/tool")]), now=NOW) + on_merge(g, [entry("alice/tool-ng")], sleep=lambda s: None) + # Someone lists a new repo under the old name; it gets its own thread. + on_merge(g, [entry("alice/tool-ng"), entry("alice/tool", issue=2)], sleep=lambda s: None) + self.assertEqual(state["discussions.json"]["alice/tool"], 99) + # It comes off the list again while the rename record is still fresh. + on_merge(g, [entry("alice/tool-ng")], sleep=lambda s: None) + self.assertEqual(state["discussions.json"]["alice/tool-ng"], 5) + + def test_completed_rename_record_is_not_replayed(self): + from scripts.on_merge import _follow + self.assertEqual(_follow({"b/x": 5, "a/y": 9}, {}, {"b/x"}), {"b/x": 5, "a/y": 9}) + + def test_no_retitle_when_both_rename_writes_fail(self): + titles = {5: "alice/tool"} + r, state = store({"discussions.json": {"alice/tool": 5}}, threads(titles), + conflicts={"discussions.json": 3, "renames.json": 3}) + g = FakeGitHub(r) + with self.assertRaises(SystemExit): + finish(g, self.plan([Action("rename", "alice/tool-ng", "", "alice/tool"), + Action("warn", "alice/tool-ng", "Heads up", "alice/tool")]), now=NOW) + self.assertEqual(titles, {5: "alice/tool"}) + self.assertEqual([c["body"] for c in self.gql(g, "CommentOnEntry")], ["Heads up"]) - def list_threads(_): - nodes = [{"number": n, "title": t, "author": {"login": "devstreamhq-bot[bot]"}} - for n, t in sorted(titles.items())] - return {"repository": {"discussions": {**listing["repository"]["discussions"], "nodes": nodes}}} + def test_drop_retitled_only_after_its_key_is_gone(self): + titles = {6: "bob/kept"} + r, state = store({"discussions.json": {"bob/kept": 6}}, threads(titles), conflicts={"discussions.json": 3}) + g = FakeGitHub(r) + with self.assertRaises(SystemExit): + finish(g, self.plan([Action("drop", "bob/kept", "Merged", "bob/kept")]), now=NOW) + self.assertEqual(titles, {6: "bob/kept"}) + self.assertEqual([c["body"] for c in self.gql(g, "CommentOnEntry")], ["Merged"]) - r, state = store({"discussions.json": {"alice/tool": 5, "bob/kept": 6}}, { - **threads(titles), - "GRAPHQL LikesCategory": {"repository": {"id": "R", "discussionCategories": { - "nodes": [{"id": "C", "slug": "likes"}]}}}, - "GRAPHQL LikesDiscussions": list_threads, - }) + def test_failed_collision_move_is_finished_by_the_next_on_merge(self): + from scripts.on_merge import run as on_merge + + titles = {5: "alice/tool", 6: "bob/kept"} + r, state = store({"discussions.json": {"alice/tool": 5, "bob/kept": 6}}, self.on_merge_routes(titles), + conflicts={"discussions.json": 3}) g = FakeGitHub(r) - finish(g, {"changed": True, "actions": [a.__dict__ for a in actions], "rot_state": rot}) - self.assertEqual(titles, {5: "bob/kept", 6: "Removed: bob/kept"}) - self.assertEqual(state["discussions.json"], {"bob/kept": 5}) - dropped = normalize_repo(titles[6]) - self.assertNotIn(dropped, {normalize_repo(e["repo"]) for e in new + [older, newer]}) - self.assertIsNone(REPO_RE.fullmatch(dropped)) - - on_merge(g, new, sleep=lambda s: None) + with self.assertRaises(SystemExit): + finish(g, self.plan([Action("rename", "bob/kept", "", "alice/tool"), + Action("drop", "bob/kept", "Merged", "bob/kept")]), now=NOW) + # The drop's key wasn't removed, so its thread keeps its title until a maintainer looks. + self.assertEqual(titles, {5: "bob/kept", 6: "bob/kept"}) + # The stored bob/kept is the dropped entry's thread; the kept entry's Likes are on 5. + on_merge(g, [entry("bob/kept")], sleep=lambda s: None) self.assertEqual(state["discussions.json"]["bob/kept"], 5) + self.assertNotIn("alice/tool", state["discussions.json"]) self.assertEqual(self.gql(g, "CreateEntryDiscussion"), [])