From 6bc18fc6f7afc080b5bbb2350620196423dddcdf Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:03:58 -0700 Subject: [PATCH 1/6] fix: stop Discussion listing looping on a repeated cursor A repeated or missing endCursor now raises instead of refetching the same page until the job times out, and a REST page that isn't a list raises instead of yielding a dict's keys as items. Multi-page tests for both. Closes #43 --- scripts/devstream/discussions.py | 6 +++++- scripts/devstream/gh.py | 6 +++++- scripts/tests/fake_gh.py | 2 ++ scripts/tests/test_discussions.py | 31 ++++++++++++++++++++++++++++++- scripts/tests/test_gh.py | 9 +++++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/scripts/devstream/discussions.py b/scripts/devstream/discussions.py index 024e072..2d3210f 100644 --- a/scripts/devstream/discussions.py +++ b/scripts/devstream/discussions.py @@ -31,7 +31,7 @@ def list_discussions(gh, category_id: str) -> dict[str, int]: """Every thread already in the category, keyed by normalised repo.""" - out, cursor = {}, None + out, cursor, seen = {}, None, set() while True: page = gh.graphql(_LIST, {"cursor": cursor, "cat": category_id})["repository"]["discussions"] for n in page["nodes"]: @@ -45,6 +45,10 @@ def list_discussions(gh, category_id: str) -> dict[str, int]: if not page["pageInfo"]["hasNextPage"]: return out cursor = page["pageInfo"]["endCursor"] + # A cursor seen before would loop until the job times out. + if not cursor or cursor in seen: + raise GitHubError(200, "discussions cursor repeated") + seen.add(cursor) def likes_category(gh) -> tuple[str, str]: diff --git a/scripts/devstream/gh.py b/scripts/devstream/gh.py index 022b2ff..d03a47a 100644 --- a/scripts/devstream/gh.py +++ b/scripts/devstream/gh.py @@ -164,7 +164,11 @@ def paginate(self, path, params=None, must_exist=False) -> Iterator[dict]: raise GitHubError(404, f"page not found: {path}") first = False headers, page = r - yield from page["items"] if isinstance(page, dict) and "items" in page else page + items = page["items"] if isinstance(page, dict) and "items" in page else page + if not isinstance(items, list): + # A dict here would yield its keys as if they were items. + raise GitHubError(200, f"not a list page: {path}") + yield from items m = re.search(r'<([^>]+)>\s*;\s*rel="next"', headers.get("Link") or "") # Rebuilt against base so the token never leaves the API host. nxt = urllib.parse.urlsplit(m.group(1)) if m else None diff --git a/scripts/tests/fake_gh.py b/scripts/tests/fake_gh.py index db9c4ab..3a48276 100644 --- a/scripts/tests/fake_gh.py +++ b/scripts/tests/fake_gh.py @@ -107,6 +107,8 @@ def paginate(self, path, params=None, must_exist=False): raise GitHubError(404, f"page not found: {path}") if isinstance(page, dict) and "items" in page: page = page["items"] + if page is not None and not isinstance(page, list): + raise GitHubError(200, f"not a list page: {path}") yield from page or [] def _write(self, method, path, body): diff --git a/scripts/tests/test_discussions.py b/scripts/tests/test_discussions.py index 162cb72..85702ad 100644 --- a/scripts/tests/test_discussions.py +++ b/scripts/tests/test_discussions.py @@ -1,6 +1,6 @@ import unittest -from scripts.devstream.discussions import ensure_discussion, likes_category +from scripts.devstream.discussions import ensure_discussion, likes_category, list_discussions from scripts.devstream.gh import GitHubError from scripts.tests.fake_gh import FakeGitHub @@ -56,5 +56,34 @@ def test_missing_raises(self): self.assertIn("Discussion category 'likes' is missing", c.exception.message) +class Listing(unittest.TestCase): + def pages(self, cursors): + """One thread per page; cursors[i] is the endCursor page i hands back, None on the last.""" + def page(v): + i = 0 if v["cursor"] is None else cursors.index(v["cursor"]) + 1 + return {"repository": {"discussions": { + "nodes": [{"number": i + 1, "title": f"o/r{i}", "author": {"login": "devstreamhq-bot[bot]"}}], + "pageInfo": {"hasNextPage": cursors[i] is not None, "endCursor": cursors[i]}}}} + return FakeGitHub({"GRAPHQL LikesDiscussions": page}) + + def test_follows_every_page(self): + g = self.pages(["c1", "c2", None]) + self.assertEqual(list_discussions(g, "C"), {"o/r0": 1, "o/r1": 2, "o/r2": 3}) + self.assertEqual([v["cursor"] for _, _, v in g.calls], [None, "c1", "c2"]) + + def test_repeated_cursor_raises(self): + g = FakeGitHub({"GRAPHQL LikesDiscussions": {"repository": {"discussions": { + "nodes": [], "pageInfo": {"hasNextPage": True, "endCursor": "stuck"}}}}}) + with self.assertRaises(GitHubError): + list_discussions(g, "C") + self.assertEqual(len(g.calls), 2) + + def test_missing_cursor_raises(self): + g = FakeGitHub({"GRAPHQL LikesDiscussions": {"repository": {"discussions": { + "nodes": [], "pageInfo": {"hasNextPage": True, "endCursor": None}}}}}) + with self.assertRaises(GitHubError): + list_discussions(g, "C") + + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_gh.py b/scripts/tests/test_gh.py index 70dd4e5..f2e0a49 100644 --- a/scripts/tests/test_gh.py +++ b/scripts/tests/test_gh.py @@ -78,6 +78,9 @@ def do_GET(self): elif p == "/xlink": nxt = f"; rel=\"next\"" self._send(200, [{"n": 1}], [("Link", nxt)]) + elif p == "/n1": + nxt = f"; rel=\"next\"" + self._send(200, [{"n": 1}], [("Link", nxt)]) elif p == "/twice": self._send(429, {}, [("retry-after", "1")]) elif p == "/q1": @@ -189,6 +192,12 @@ def test_500_is_error(self): def test_paginate_follows_link_header(self): self.assertEqual([i["n"] for i in self.gh.paginate("/p1")], [1, 2, 3]) + def test_paginate_non_list_page_raises(self): + # Iterating a dict would yield its keys as if they were items. + for path in ("/repositories/123", "/n1"): + with self.assertRaises(GitHubError): + list(self.gh.paginate(path)) + def test_paginate_search_items(self): self.assertEqual(list(self.gh.paginate("/search")), [{"n": 9}]) From 2d8bf319afa0383d0bea48680e06c8d1e7f1c2de Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:05:15 -0700 Subject: [PATCH 2/6] fix: stop odd GitHub answers crashing the API client A redirect with no Location raises a GitHubError, and a relative one is resolved against the request. Retry-After as an HTTP date is parsed, and an unreadable one waits a minute. A network error left after the retries is raised as a GitHubError, so callers that skip one item on API errors skip it on these too; which requests retry is unchanged. A GraphQL error entry that isn't an object raises instead of crashing, and the test fake stops a redirect cycle after the real client's hop limit. --- scripts/devstream/gh.py | 22 ++++++++++--- scripts/tests/fake_gh.py | 9 ++++-- scripts/tests/test_gh.py | 70 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/scripts/devstream/gh.py b/scripts/devstream/gh.py index d03a47a..743dc40 100644 --- a/scripts/devstream/gh.py +++ b/scripts/devstream/gh.py @@ -6,6 +6,8 @@ import urllib.error import urllib.parse import urllib.request +from datetime import timezone +from email.utils import parsedate_to_datetime from typing import Iterator MAX_SLEEP = 600 @@ -40,6 +42,8 @@ def redirect_request(self, *a, **k): def _allowed(e: dict, not_found: bool, forbidden: bool) -> bool: # A FORBIDDEN with no path is the whole query refused, never one hidden repo. + if not isinstance(e, dict): + return False return (not_found and e.get("type") == "NOT_FOUND") or (forbidden and e.get("type") == "FORBIDDEN" and bool(e.get("path"))) @@ -79,7 +83,14 @@ def _rate_limit_reset(status, headers) -> int | None: if headers.get("x-ratelimit-remaining") != "0" and retry is None: return None if retry is not None: - return int(time.time()) + int(retry) + if retry.strip().isdigit(): + return int(time.time()) + int(retry) + # Retry-After may also be an HTTP date; anything unreadable waits a minute. + try: + at = parsedate_to_datetime(retry) + except (TypeError, ValueError): + return int(time.time()) + 60 + return int((at if at.tzinfo else at.replace(tzinfo=timezone.utc)).timestamp()) return int(headers.get("x-ratelimit-reset") or time.time() + 60) @staticmethod @@ -109,7 +120,8 @@ def _request(self, method, path, params=None, body=None, read=None): except NETWORK as e: fails += 1 if fails >= ATTEMPTS or not self._retryable(method, read, exc=e): - raise + # As a GitHubError, the callers' per-item error handling applies to it too. + raise GitHubError(0, f"network error: {e}") from e self._backoff(fails) continue if self._retryable(method, read, status) and fails + 1 < ATTEMPTS: @@ -128,7 +140,9 @@ def _request(self, method, path, params=None, body=None, read=None): hops += 1 if hops > MAX_HOPS: raise GitHubError(status, "too many redirects") - loc = urllib.parse.urlsplit(headers["Location"]) + if not headers.get("Location"): + raise GitHubError(status, "redirect without a Location") + loc = urllib.parse.urlsplit(urllib.parse.urljoin(self.base + path, headers["Location"])) path = loc.path + (f"?{loc.query}" if loc.query else "") self.last_redirect = path continue @@ -205,7 +219,7 @@ def graphql(self, query: str, variables: dict, allow_not_found: bool = False, raise GitHubError(404, "graphql endpoint not found") out = r[1] or {} errors = out.get("errors") - if any(e.get("type") == "RATE_LIMITED" for e in errors or []): + if any(isinstance(e, dict) and e.get("type") == "RATE_LIMITED" for e in errors or []): raise RateLimited(int(r[0].get("x-ratelimit-reset") or time.time() + 60)) if errors and not all(_allowed(e, allow_not_found, allow_forbidden) for e in errors): raise GitHubError(200, json.dumps(out["errors"])[:500]) diff --git a/scripts/tests/fake_gh.py b/scripts/tests/fake_gh.py index 3a48276..70d65b6 100644 --- a/scripts/tests/fake_gh.py +++ b/scripts/tests/fake_gh.py @@ -1,7 +1,7 @@ import re import urllib.parse -from scripts.devstream.gh import GitHubError, _allowed +from scripts.devstream.gh import MAX_HOPS, GitHubError, _allowed class Redirect: @@ -62,7 +62,7 @@ def __init__(self, routes: dict[str, object]): self.calls: list[tuple[str, str, object]] = [] self.last_redirect: str | None = None - def _find(self, method, path, params, body, write=False): + def _find(self, method, path, params, body, write=False, hops=0): full = path if params: full += ("&" if "?" in path else "?") + urllib.parse.urlencode(params) @@ -72,8 +72,11 @@ def _find(self, method, path, params, body, write=False): if key in self.routes: v = self.routes[key] if isinstance(v, Redirect): + # Same limit as the real client, so a cycle fails instead of recursing forever. + if hops >= MAX_HOPS: + raise GitHubError(301, "too many redirects") self.last_redirect = v.path - return self._find(method, v.path, None, body, write) + return self._find(method, v.path, None, body, write, hops + 1) return v(body) if callable(v) else v if write: raise GitHubError(404, f"no route: {method} {path}") diff --git a/scripts/tests/test_gh.py b/scripts/tests/test_gh.py index f2e0a49..6c712cc 100644 --- a/scripts/tests/test_gh.py +++ b/scripts/tests/test_gh.py @@ -4,6 +4,7 @@ import urllib.error import time import unittest +from email.utils import formatdate from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from scripts.devstream.gh import GitHub, GitHubError, RateLimited @@ -360,7 +361,7 @@ def test_graphql_query_timeout_retried(self): def test_graphql_mutation_502_not_retried(self): for err in (502, TimeoutError()): gh, sent, _ = self.gh(err, 200) - with self.assertRaises((GitHubError, TimeoutError)): + with self.assertRaises(GitHubError): gh.graphql("mutation M { x }", {}) self.assertEqual(len(sent), 1) @@ -395,7 +396,7 @@ def test_get_network_error_retried(self): def test_three_network_errors_raise(self): gh, sent, _ = self.gh(TimeoutError(), TimeoutError(), TimeoutError(), 200) - with self.assertRaises(TimeoutError): + with self.assertRaises(GitHubError): gh.get("/a") self.assertEqual(len(sent), 3) @@ -408,7 +409,7 @@ def test_write_retried_only_when_never_sent(self): self.assertEqual(len(sent), 2) for err in (TimeoutError(), ConnectionResetError(), urllib.error.URLError("reset")): gh, sent, _ = self.gh(err, 200) - with self.assertRaises(OSError): + with self.assertRaises(GitHubError): gh.patch("/a", {}) self.assertEqual(len(sent), 1) @@ -421,7 +422,70 @@ def test_rate_limit_rules_unchanged(self): self.assertEqual(len(sent), 1) +class OddAnswers(unittest.TestCase): + """_send is scripted with (status, headers, body) answers.""" + + def gh(self, *script): + slept = [] + gh = GitHub("tok", "http://x", sleep=slept.append) + sent = [] + + def send(method, url, body): + sent.append(url) + return script[len(sent) - 1] + gh._send = send + return gh, sent, slept + + def test_redirect_without_location_raises(self): + gh, sent, _ = self.gh((301, {}, b"")) + with self.assertRaises(GitHubError) as c: + gh.get("/repos/a/b") + self.assertEqual(c.exception.status, 301) + + def test_relative_redirect_followed(self): + for loc, want in (("/repositories/9", "/repositories/9"), ("9", "/repositories/9")): + gh, sent, _ = self.gh((301, {"Location": loc}, b""), (200, {}, b'{"ok": 1}')) + self.assertEqual(gh.get("/repositories/old"), {"ok": 1}) + self.assertEqual((sent[1], gh.last_redirect), ("http://x" + want, want)) + + def test_retry_after_http_date(self): + gh, sent, slept = self.gh((429, {"retry-after": formatdate(time.time() + 30, usegmt=True)}, b"{}"), + (200, {}, b'{"ok": 1}')) + self.assertEqual(gh.get("/a"), {"ok": 1}) + self.assertAlmostEqual(slept[0], 30, delta=3) + gh, _, _ = self.gh((429, {"retry-after": formatdate(LONG, usegmt=True)}, b"{}")) + with self.assertRaises(RateLimited) as c: + gh.get("/a") + self.assertEqual(c.exception.reset_at, LONG) + + def test_retry_after_garbage_waits_a_minute(self): + gh, _, slept = self.gh((429, {"retry-after": "soon"}, b"{}"), (200, {}, b'{"ok": 1}')) + self.assertEqual(gh.get("/a"), {"ok": 1}) + self.assertAlmostEqual(slept[0], 60, delta=3) + + def test_graphql_error_that_is_not_a_dict(self): + for errors in (["boom"], [None], [{"type": "NOT_FOUND", "path": ["r0"]}, "boom"]): + gh, _, _ = self.gh((200, {}, json.dumps({"data": {"r0": None}, "errors": errors}).encode())) + with self.assertRaises(GitHubError): + gh.graphql("query Q { x }", {}, allow_not_found=True) + + def test_network_error_after_retries_is_a_github_error(self): + gh = GitHub("tok", "http://x", sleep=lambda s: None) + + def send(*a): + raise urllib.error.URLError("down") + gh._send = send + with self.assertRaises(GitHubError) as c: + gh.get("/a") + self.assertIsInstance(c.exception.__cause__, urllib.error.URLError) + + class FakeTest(unittest.TestCase): + def test_redirect_cycle_raises(self): + f = FakeGitHub({"GET /a": Redirect("/b"), "GET /b": Redirect("/a")}) + with self.assertRaises(GitHubError): + f.get("/a") + def test_same_public_methods(self): pub = lambda c: {n for n in dir(c) if not n.startswith("_") and callable(getattr(c, n))} self.assertEqual(pub(GitHub), pub(FakeGitHub)) From 6e92ec1d6af214a1b0aa2461b66cc1104212e26c Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:06:41 -0700 Subject: [PATCH 3/6] fix: stop lost Like-burst records reopening issues every hour A run whose flags.json write fails after it opened issues now leaves a way back: the next run adopts the bot's own open Like issues, and its open aggregate, by title instead of opening them again. A failed aggregate PATCH is logged and the sync goes on; an aggregate closed or deleted mid-edit gets a fresh issue for the new entries, so they aren't marked reviewed unseen. One failed issue lookup skips only that record. Closes #39 --- scripts/sync_likes.py | 34 ++++++++++++++-- scripts/tests/test_likes.py | 80 ++++++++++++++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 4 deletions(-) diff --git a/scripts/sync_likes.py b/scripts/sync_likes.py index 2eb5116..86a6dfd 100644 --- a/scripts/sync_likes.py +++ b/scripts/sync_likes.py @@ -31,6 +31,8 @@ WEEK = timedelta(days=7) ISSUE_CAP = 5 LABELS = ["report", "needs-review"] +BOT = "devstreamhq-bot[bot]" +AGGREGATE_TITLE = re.compile(r"Like burst: [0-9]+ entr(y|ies)") # How long a reviewed (closed) issue holds off a new one for the same entry. COOLDOWN = {"burst": DAY, "drip": WEEK} SNAPSHOTS_KEPT = 30 @@ -67,7 +69,9 @@ def _review(gh, flags: dict, now: datetime) -> None: issue = gh.get(f"{API}/issues/{n}") except GitHubError as e: if e.status != 410: # 410: deleted - raise + # Its record stays pending and is looked up again next hour; the rest still sync. + logging.warning("could not look up issue #%s: %s", n, e) + continue issue = None if issue is None or issue.get("state") == "closed": done.add(n) @@ -145,10 +149,20 @@ def _baseline(gh, now: datetime, renames: dict) -> dict | None: return data +def _open_reports(gh) -> dict[str, int]: + """The bot's open Like issues by title. Only the bot can author these, so a title can't be forged here.""" + return {i["title"]: i["number"] for i in gh.paginate(f"{API}/issues", {"state": "open", "labels": LABELS[0], + "per_page": 100}) + if (i.get("user") or {}).get("login") == BOT and "pull_request" not in i} + + def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetime) -> None: """todo is (kind, key, repo, body). Records are set right after each post, so a failure keeps them.""" + # A failed flags.json write loses the records of issues already open; adopt those rather than open them again. + opened = _open_reports(gh) for kind, key, repo, body in todo[:ISSUE_CAP]: - n = gh.post(f"{API}/issues", {"title": f"Like {kind}: {repo}", "body": body, "labels": LABELS})["number"] + title = f"Like {kind}: {repo}" + n = opened.get(title) or gh.post(f"{API}/issues", {"title": title, "body": body, "labels": LABELS})["number"] flags.setdefault(key, {})[kind] = _record(kind, n, now) overflow = [(key, kind) for kind, key, _, _ in todo[ISSUE_CAP:]] if not overflow: @@ -156,6 +170,8 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim # A flood of decoys must not become a flood of issues, nor let the target slip its flag. agg = next((r["issue"] for recs in flags.values() for r in recs.values() if r.get("aggregate") and r["reviewed_at"] is None), None) + if agg is None: + agg = next((n for t, n in opened.items() if AGGREGATE_TITLE.fullmatch(t)), None) if agg is None: agg = gh.post(f"{API}/issues", {**_aggregate(overflow), "labels": LABELS})["number"] for key, kind in overflow: @@ -165,7 +181,19 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim flags.setdefault(key, {})[kind] = _record(kind, agg, now, aggregate=True) listed = [(k, kind) for k, recs in flags.items() for kind, r in recs.items() if r["issue"] == agg and r["reviewed_at"] is None] - gh.patch(f"{API}/issues/{agg}", _aggregate(listed)) + try: + state = (gh.patch(f"{API}/issues/{agg}", _aggregate(listed)) or {}).get("state") + except GitHubError as e: + if e.status not in (404, 410): + # The records already point at it, so the next overflow brings the body up to date. + logging.warning("could not update aggregate issue #%s: %s", agg, e) + return + state = "closed" + if state == "closed": + # Closed or deleted since _review: its review is over, so these entries need an issue of their own. + agg = gh.post(f"{API}/issues", {**_aggregate(overflow), "labels": LABELS})["number"] + for key, kind in overflow: + flags[key][kind] = _record(kind, agg, now, aggregate=True) def _merge_flags(flags: dict, before: dict): diff --git a/scripts/tests/test_likes.py b/scripts/tests/test_likes.py index f261993..d109478 100644 --- a/scripts/tests/test_likes.py +++ b/scripts/tests/test_likes.py @@ -58,6 +58,7 @@ def __init__(self, entries, discussions, users=None, stats=None, stars=None, iss "GRAPHQL DiscussionReactors": self._reactors, "GRAPHQL RepoMeta": self._meta, f"POST {API}/issues": self._open_issue, + f"GET {API}/issues": self._list_issues, } for login, created in users.items(): routes[f"GET /users/{login}"] = { @@ -115,10 +116,15 @@ def _meta(self, v): def _open_issue(self, body): n = self.next_issue self.next_issue += 1 - self.routes[f"GET {API}/issues/{n}"] = {"number": n, "state": "open", "title": body["title"]} + self.routes[f"GET {API}/issues/{n}"] = {"number": n, "state": "open", "title": body["title"], + "user": {"login": "devstreamhq-bot[bot]"}} self.routes[f"PATCH {API}/issues/{n}"] = {} return {"number": n} + def _list_issues(self, _): + return [{"user": {"login": "mallory"}, **v} for k, v in self.routes.items() + if k.startswith(f"GET {API}/issues/") and isinstance(v, dict) and v.get("state") == "open"] + def close(self, n): self.routes[f"GET {API}/issues/{n}"]["state"] = "closed" @@ -496,6 +502,78 @@ def boom(_): self.assertEqual(w.issues_opened(), []) self.assertTrue(w.written("likes.json")["o/a"]["flagged"]) + def test_failed_flags_write_after_issues_opens_no_duplicates(self): + repos = {f"o/r{j}": burst_nodes(10, 10) for j in range(6)} + w = World([entry(r) for r in repos], repos) + + def boom(_): + raise GitHubError(502, "bad gateway") + w.routes[f"PUT {API}/contents/flags.json"] = boom + with self.assertRaises(GitHubError): + w.run() + self.assertEqual(len(w.issues_opened()), 6) + w.persist() + w.routes.pop(f"GET {API}/contents/flags.json") # that PUT never landed + w.routes[f"PUT {API}/contents/flags.json"] = {} + w.run(now=NOW + timedelta(minutes=5)) + # The lost records are rebuilt from the bot's open issues, not opened again. + self.assertEqual(w.issues_opened(), []) + flags = w.written("flags.json") + self.assertEqual(sorted(r["burst"]["issue"] for r in flags.values()), [100, 101, 102, 103, 104, 105]) + + def test_aggregate_patch_failure_keeps_the_run(self): + first = {f"o/r{j}": burst_nodes(10, 10) for j in range(6)} + later = {f"o/s{j}": [] for j in range(6)} + w = World([entry(r) for r in {**first, **later}], {**first, **later}) + w.run() + agg = w.next_issue - 1 + w.persist() + + def boom(_): + raise GitHubError(502, "bad gateway") + w.routes[f"PATCH {API}/issues/{agg}"] = boom + for r in later: + w.set_nodes(r, burst_nodes(10, 10, reacted=iso(NOW + timedelta(minutes=30)))) + with self.assertLogs(level="WARNING"): + w.run(now=NOW + timedelta(hours=1)) + self.assertEqual(w.written("flags.json")["o/s5"]["burst"]["issue"], agg) + self.assertIn("o/s5", w.written("likes.json")) + + def test_aggregate_closed_mid_edit_gets_a_new_issue(self): + first = {f"o/r{j}": burst_nodes(10, 10) for j in range(6)} + later = {f"o/s{j}": [] for j in range(6)} + for gone in ({"state": "closed"}, None): + w = World([entry(r) for r in {**first, **later}], {**first, **later}) + w.run() + agg = w.next_issue - 1 + w.persist() + if gone is None: + del w.routes[f"PATCH {API}/issues/{agg}"] # deleted: the fake answers 404 + else: + w.routes[f"PATCH {API}/issues/{agg}"] = gone + for r in later: + w.set_nodes(r, burst_nodes(10, 10, reacted=iso(NOW + timedelta(minutes=30)))) + w.run(now=NOW + timedelta(hours=1)) + self.assertEqual(titles(w)[-1], "Like burst: 1 entry") + self.assertEqual(w.written("flags.json")["o/s5"]["burst"]["issue"], w.next_issue - 1) + self.assertNotEqual(w.next_issue - 1, agg) + + def test_one_failed_review_lookup_skips_only_that_issue(self): + stats = {"flags.json": {r: {"burst": {"kind": "burst", "issue": n, "opened_at": iso(NOW), + "reviewed_at": None}} for r, n in (("o/a", 7), ("o/b", 8))}} + w = World([entry("o/a"), entry("o/b")], {"o/a": [], "o/b": []}, stats=stats, + issues=[{"number": 8, "state": "closed"}]) + + def boom(_): + raise GitHubError(502, "bad gateway") + w.routes[f"GET {API}/issues/7"] = boom + with self.assertLogs(level="WARNING"): + w.run() + flags = w.written("flags.json") + self.assertIsNone(flags["o/a"]["burst"]["reviewed_at"]) + self.assertEqual(flags["o/b"]["burst"]["reviewed_at"], iso(NOW)) + self.assertTrue(w.written("likes.json")["o/a"]["flagged"]) + def test_failed_issue_post_keeps_flags_for_opened_ones(self): repos = {f"o/r{j}": burst_nodes(10, 10) for j in range(2)} w = World([entry(r) for r in repos], repos) From 9566052febe14b60259906b8a6adcd8dfcb478bd Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:07:56 -0700 Subject: [PATCH 4/6] fix: stop a failed rename key move splitting an entry's Likes When rot's finish couldn't move a renamed entry's key in discussions.json, nothing tried again, and the hourly on_merge opened a second thread for the new name. Finish now records renames.json before the key move and retitles the thread either way; on_merge replays renames.json onto the mapping before creating anything, so the next run finishes the move, including after a rename collision. An old name that is listed again keeps its own key. Closes #37 --- scripts/on_merge.py | 20 ++++++++++-- scripts/rot.py | 13 ++++---- scripts/sync_likes.py | 2 +- scripts/tests/test_rot.py | 69 ++++++++++++++++++++++++++++++++++----- 4 files changed, 86 insertions(+), 18 deletions(-) diff --git a/scripts/on_merge.py b/scripts/on_merge.py index 04b4d81..82c745d 100644 --- a/scripts/on_merge.py +++ b/scripts/on_merge.py @@ -12,6 +12,18 @@ VISIBLE = {"live", "warned", "flagged"} +def _follow(mapping: dict, renames: dict, listed: set[str]) -> dict: + """Finish a rename whose key move failed in rot's finish, so the entry keeps its thread and its Likes. + + Overrides a stored new key: that is the dropped entry's thread after a rename collision. + An old name listed again is someone else's entry and keeps its own key. + """ + for old, r in sorted(((k, r) for k, r in renames.items() if isinstance(r, dict)), key=lambda kv: kv[1]["at"]): + if old in mapping and old not in listed: + mapping[normalize_repo(r["to"])] = mapping.pop(old) + return mapping + + def run(gh, entries, sleep=time.sleep): ensure_branch(gh) cur = read_json(gh, "discussions.json")[0] @@ -22,6 +34,10 @@ def run(gh, entries, sleep=time.sleep): # setdefault: a stored mapping always wins, so a thread retitled onto a listed name can't take it over. for k, v in list_discussions(gh, category_id).items(): mapping.setdefault(k, v) + # After adopting, so a thread still titled with the old name can't put the old key back. + renames = read_json(gh, "renames.json")[0] or {} + listed = {normalize_repo(e["repo"]) for e in entries} + _follow(mapping, renames, listed) todo = {} for e in entries: if e["status"] in VISIBLE and normalize_repo(e["repo"]) not in mapping: @@ -40,8 +56,8 @@ def run(gh, entries, sleep=time.sleep): added = {k: v for k, v in mapping.items() if before.get(k) != v} def merge(stored): - # Re-applied onto a fresh read after a 409; a stored mapping still wins. - return {**added, **{normalize_repo(k): v for k, v in stored.items()}} + # Re-applied onto a fresh read after a 409; a stored mapping still wins, once renames are followed. + return {**added, **_follow({normalize_repo(k): v for k, v in stored.items()}, renames, listed)} if mapping != before: try: update_json(gh, "discussions.json", merge, "chore(stats): map entry discussions") diff --git a/scripts/rot.py b/scripts/rot.py index dfe06f7..8f74a6c 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -346,24 +346,23 @@ def record(data): numbers = dict(read_json(gh, "discussions.json")[0] or {}) # before the move, so dropped entries find their thread failed = 0 - moved = True for path, mutate, message in ( ("rot.json", lambda _: plan["rot_state"], "chore(stats): record failing repos"), + # Before the key move: if that write fails, on_merge finishes it from this record. + ("renames.json", record, "chore(stats): record renamed repos"), ("discussions.json", rekey, "chore(stats): follow renamed repos"), ("flags.json", rekey, "chore(stats): follow renamed repos"), # The site build reads these two by key, so they move now rather than at the next hourly sync. ("likes.json", rekey, "chore(stats): follow renamed repos"), ("meta.json", rekey, "chore(stats): follow renamed repos"), - ("renames.json", record, "chore(stats): record renamed repos"), ): try: update_json(gh, path, mutate, message) except Exception: logging.exception("could not write %s", path) failed += 1 - moved = moved and path != "discussions.json" - # A retitle without its key move would leave the thread named for a key it isn't stored under. - posted, notified = _post(gh, actions, numbers, retitle=moved) + # Retitled even if the key move failed: on_merge moves the key from renames.json, and adopts by title. + posted, notified = _post(gh, actions, numbers) failed += posted + _mark_notified(gh, notified, plan["rot_state"]) if failed: raise SystemExit(f"{failed} Discussion or stats update(s) failed") @@ -392,7 +391,7 @@ def mark(data): return 0 -def _post(gh, actions, numbers: dict, retitle: bool = True) -> tuple[int, set[str]]: +def _post(gh, actions, numbers: dict) -> tuple[int, set[str]]: """Comment on, or retitle, each action's Discussion. numbers maps stats key to Discussion number. Returns (failures, rot keys whose notice is done). @@ -408,7 +407,7 @@ def _post(gh, actions, numbers: dict, retitle: bool = True) -> tuple[int, set[st if d is not None: if a.kind != "rename": gh.graphql(_COMMENT, {"discussionId": d["id"], "body": a.message}) - if a.kind in ("rename", "drop") and retitle: + if a.kind in ("rename", "drop"): # "Removed: " can never normalise to a listed repo, so on_merge won't adopt the thread. title = a.repo if a.kind == "rename" else f"{REMOVED_PREFIX}{a.repo}" gh.graphql(_RETITLE, {"discussionId": d["id"], "title": title}) diff --git a/scripts/sync_likes.py b/scripts/sync_likes.py index 86a6dfd..0b31224 100644 --- a/scripts/sync_likes.py +++ b/scripts/sync_likes.py @@ -212,7 +212,7 @@ def mutate(cur): def _rename_in_flight(gh, now, entries, keys, mapping, prev_likes) -> bool: - """True while a rename has merged but rot's finish hasn't moved discussions.json yet. + """True while a rename has merged but discussions.json hasn't moved yet (rot's finish, or on_merge within the hour). The tell: a listed key has no thread, and a key that was ranked last time has a thread but is no longer listed. An entry removed for good drops out of likes.json, so it can't hold the board. diff --git a/scripts/tests/test_rot.py b/scripts/tests/test_rot.py index 4dce5f3..039250e 100644 --- a/scripts/tests/test_rot.py +++ b/scripts/tests/test_rot.py @@ -674,18 +674,71 @@ def test_failed_rot_write_never_starts_the_clock_on_an_old_record(self): new, actions, _ = plan_rot([entry()], {"alice/tool": gone()}, NOW + timedelta(days=1), state["rot.json"]) self.assertEqual((new, kinds(actions)), ([entry()], [("notice", "alice/tool")])) - def test_no_retitle_when_the_key_move_fails(self): - titles = {5: "alice/tool", 6: "bob/kept"} - r, state = store({"discussions.json": {"alice/tool": 5, "bob/kept": 6}}, threads(titles), + def on_merge_routes(self, titles): + def list_threads(_): + nodes = [{"number": n, "title": t, "author": {"login": "devstreamhq-bot[bot]"}} + for n, t in sorted(titles.items())] + return {"repository": {"discussions": { + "nodes": nodes, "pageInfo": {"hasNextPage": False, "endCursor": None}}}} + return { + **threads(titles), + "GRAPHQL LikesCategory": {"repository": {"id": "R", "discussionCategories": { + "nodes": [{"id": "C", "slug": "likes"}]}}}, + "GRAPHQL LikesDiscussions": list_threads, + "GRAPHQL CreateEntryDiscussion": {"createDiscussion": {"discussion": {"number": 99}}}, + } + + def test_failed_key_move_is_finished_by_the_next_on_merge(self): + from scripts.on_merge import run as on_merge + + titles = {5: "alice/tool", 6: "bob/other"} + r, state = store({"discussions.json": {"alice/tool": 5, "bob/other": 6}}, self.on_merge_routes(titles), conflicts={"discussions.json": 3}) g = FakeGitHub(r) with self.assertRaises(SystemExit): finish(g, self.plan([Action("rename", "alice/tool-ng", "", "alice/tool"), - Action("drop", "bob/kept", "Merged", "bob/kept"), - Action("warn", "alice/tool-ng", "Heads up", "alice/tool")])) - self.assertEqual(self.gql(g, "UpdateEntryDiscussion"), []) - self.assertEqual(titles, {5: "alice/tool", 6: "bob/kept"}) - self.assertEqual(sorted(c["body"] for c in self.gql(g, "CommentOnEntry")), ["Heads up", "Merged"]) + Action("warn", "alice/tool-ng", "Heads up", "alice/tool")]), now=NOW) + self.assertIn("alice/tool", state["discussions.json"]) + self.assertEqual(state["renames.json"], {"alice/tool": {"to": "alice/tool-ng", "at": iso(NOW)}}) + self.assertEqual(titles[5], "alice/tool-ng") + self.assertEqual([c["body"] for c in self.gql(g, "CommentOnEntry")], ["Heads up"]) + + listed = [entry("alice/tool-ng"), entry("bob/other", issue=2)] + on_merge(g, listed, sleep=lambda s: None) + self.assertEqual(state["discussions.json"], {"alice/tool-ng": 5, "bob/other": 6, "other/writer": 1}) + self.assertEqual(self.gql(g, "CreateEntryDiscussion"), []) + g.calls.clear() + on_merge(g, listed, sleep=lambda s: None) + self.assertEqual([c for c in g.calls if c[0] == "PUT"], []) + + def test_failed_collision_move_is_finished_by_the_next_on_merge(self): + from scripts.on_merge import run as on_merge + + titles = {5: "alice/tool", 6: "bob/kept"} + r, state = store({"discussions.json": {"alice/tool": 5, "bob/kept": 6}}, self.on_merge_routes(titles), + conflicts={"discussions.json": 3}) + g = FakeGitHub(r) + with self.assertRaises(SystemExit): + finish(g, self.plan([Action("rename", "bob/kept", "", "alice/tool"), + Action("drop", "bob/kept", "Merged", "bob/kept")]), now=NOW) + self.assertEqual(titles, {5: "bob/kept", 6: "Removed: bob/kept"}) + # The stored bob/kept is the dropped entry's thread; the kept entry's Likes are on 5. + on_merge(g, [entry("bob/kept")], sleep=lambda s: None) + self.assertEqual(state["discussions.json"]["bob/kept"], 5) + self.assertNotIn("alice/tool", state["discussions.json"]) + self.assertEqual(self.gql(g, "CreateEntryDiscussion"), []) + + def test_no_move_onto_a_name_that_is_listed_again(self): + from scripts.on_merge import run as on_merge + + # alice/tool was renamed away, then someone listed a new repo under the old name. + titles = {5: "alice/tool-ng"} + r, state = store({"discussions.json": {"alice/tool-ng": 5}, + "renames.json": {"alice/tool": {"to": "alice/tool-ng", "at": iso(NOW)}}}, + self.on_merge_routes(titles)) + g = FakeGitHub(r) + on_merge(g, [entry("alice/tool-ng"), entry("alice/tool", issue=2)], sleep=lambda s: None) + self.assertEqual(state["discussions.json"], {"alice/tool-ng": 5, "alice/tool": 99}) def test_collision_keeps_the_older_entrys_thread_through_on_merge(self): from scripts.devstream.entries import normalize_repo From 3f8eaea2c52b95566e5e5cc28f65f67f77624d57 Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:14:32 -0700 Subject: [PATCH 5/6] fix: stop on_merge replaying renames whose key move completed Only a renames.json record that rot marked pending, because its discussions.json move failed, is replayed, and on_merge clears the mark once it has. A finished rename can no longer pull its thread back after the old name is relisted and unlisted. Threads are retitled only when their key is stored or will be: a rename needs the key move or its pending record, a drop needs its key removed. --- scripts/on_merge.py | 31 ++++++++----- scripts/rot.py | 27 ++++++----- scripts/tests/test_rot.py | 94 +++++++++++++++++++-------------------- 3 files changed, 81 insertions(+), 71 deletions(-) diff --git a/scripts/on_merge.py b/scripts/on_merge.py index 82c745d..40a691c 100644 --- a/scripts/on_merge.py +++ b/scripts/on_merge.py @@ -12,13 +12,14 @@ VISIBLE = {"live", "warned", "flagged"} -def _follow(mapping: dict, renames: dict, listed: set[str]) -> dict: +def _follow(mapping: dict, pending: dict, listed: set[str]) -> dict: """Finish a rename whose key move failed in rot's finish, so the entry keeps its thread and its Likes. + pending holds only renames.json records rot marked pending; a completed move is never replayed. Overrides a stored new key: that is the dropped entry's thread after a rename collision. An old name listed again is someone else's entry and keeps its own key. """ - for old, r in sorted(((k, r) for k, r in renames.items() if isinstance(r, dict)), key=lambda kv: kv[1]["at"]): + for old, r in sorted(pending.items(), key=lambda kv: kv[1]["at"]): if old in mapping and old not in listed: mapping[normalize_repo(r["to"])] = mapping.pop(old) return mapping @@ -35,9 +36,10 @@ def run(gh, entries, sleep=time.sleep): for k, v in list_discussions(gh, category_id).items(): mapping.setdefault(k, v) # After adopting, so a thread still titled with the old name can't put the old key back. - renames = read_json(gh, "renames.json")[0] or {} + pending = {k: r for k, r in (read_json(gh, "renames.json")[0] or {}).items() + if isinstance(r, dict) and r.get("pending")} listed = {normalize_repo(e["repo"]) for e in entries} - _follow(mapping, renames, listed) + _follow(mapping, pending, listed) todo = {} for e in entries: if e["status"] in VISIBLE and normalize_repo(e["repo"]) not in mapping: @@ -57,14 +59,21 @@ def run(gh, entries, sleep=time.sleep): def merge(stored): # Re-applied onto a fresh read after a 409; a stored mapping still wins, once renames are followed. - return {**added, **_follow({normalize_repo(k): v for k, v in stored.items()}, renames, listed)} - if mapping != before: - try: + return {**added, **_follow({normalize_repo(k): v for k, v in stored.items()}, pending, listed)} + + def done(stored): + # Replayed once: a later relist and unlist of the old name must never pull its thread back. + return {k: {f: x for f, x in v.items() if f != "pending"} if pending.get(k) == v else v + for k, v in stored.items()} + try: + if mapping != before: update_json(gh, "discussions.json", merge, "chore(stats): map entry discussions") - except Exception: - if err is None: - raise - logging.exception("could not write discussions.json") + if pending: + update_json(gh, "renames.json", done, "chore(stats): finish renamed repos") + except Exception: + if err is None: + raise + logging.exception("could not write discussions.json or renames.json") def main(): diff --git a/scripts/rot.py b/scripts/rot.py index 8f74a6c..cd423c2 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -342,15 +342,17 @@ def record(data): # sync_likes replays these over snapshots taken before the rename; past its 7-day baseline they're unused. kept = {k: v for k, v in data.items() if isinstance(v, dict) and now - _parse(v["at"]) < RENAME_KEEP} # renames.json keeps one record per old name; a repeat rename replaces it, fine on a weekly schedule. - return {**kept, **{old: {"to": new, "at": _iso(now)} for old, new in moves.items()}} + # pending: the key move failed, so on_merge replays this record once; a completed move is never replayed. + mark = {"pending": True} if "discussions.json" in failed else {} + return {**kept, **{old: {"to": new, "at": _iso(now), **mark} for old, new in moves.items()}} numbers = dict(read_json(gh, "discussions.json")[0] or {}) # before the move, so dropped entries find their thread - failed = 0 + failed = set() for path, mutate, message in ( ("rot.json", lambda _: plan["rot_state"], "chore(stats): record failing repos"), - # Before the key move: if that write fails, on_merge finishes it from this record. - ("renames.json", record, "chore(stats): record renamed repos"), ("discussions.json", rekey, "chore(stats): follow renamed repos"), + # After the key move, so record knows whether on_merge has to finish it. + ("renames.json", record, "chore(stats): record renamed repos"), ("flags.json", rekey, "chore(stats): follow renamed repos"), # The site build reads these two by key, so they move now rather than at the next hourly sync. ("likes.json", rekey, "chore(stats): follow renamed repos"), @@ -360,12 +362,13 @@ def record(data): update_json(gh, path, mutate, message) except Exception: logging.exception("could not write %s", path) - failed += 1 - # Retitled even if the key move failed: on_merge moves the key from renames.json, and adopts by title. - posted, notified = _post(gh, actions, numbers) - failed += posted + _mark_notified(gh, notified, plan["rot_state"]) - if failed: - raise SystemExit(f"{failed} Discussion or stats update(s) failed") + failed.add(path) + # A title must match a key that is stored, or will be once on_merge replays the pending record. + retitle = {"rename": not {"discussions.json", "renames.json"} <= failed, "drop": "discussions.json" not in failed} + posted, notified = _post(gh, actions, numbers, retitle) + total = len(failed) + posted + _mark_notified(gh, notified, plan["rot_state"]) + if total: + raise SystemExit(f"{total} Discussion or stats update(s) failed") def _mark_notified(gh, keys: set[str], planned: dict) -> int: @@ -391,7 +394,7 @@ def mark(data): return 0 -def _post(gh, actions, numbers: dict) -> tuple[int, set[str]]: +def _post(gh, actions, numbers: dict, retitle: dict | None = None) -> tuple[int, set[str]]: """Comment on, or retitle, each action's Discussion. numbers maps stats key to Discussion number. Returns (failures, rot keys whose notice is done). @@ -407,7 +410,7 @@ def _post(gh, actions, numbers: dict) -> tuple[int, set[str]]: if d is not None: if a.kind != "rename": gh.graphql(_COMMENT, {"discussionId": d["id"], "body": a.message}) - if a.kind in ("rename", "drop"): + if a.kind in ("rename", "drop") and (retitle or {}).get(a.kind, True): # "Removed: " can never normalise to a listed repo, so on_merge won't adopt the thread. title = a.repo if a.kind == "rename" else f"{REMOVED_PREFIX}{a.repo}" gh.graphql(_RETITLE, {"discussionId": d["id"], "title": title}) diff --git a/scripts/tests/test_rot.py b/scripts/tests/test_rot.py index 039250e..709a68f 100644 --- a/scripts/tests/test_rot.py +++ b/scripts/tests/test_rot.py @@ -699,79 +699,77 @@ def test_failed_key_move_is_finished_by_the_next_on_merge(self): finish(g, self.plan([Action("rename", "alice/tool-ng", "", "alice/tool"), Action("warn", "alice/tool-ng", "Heads up", "alice/tool")]), now=NOW) self.assertIn("alice/tool", state["discussions.json"]) - self.assertEqual(state["renames.json"], {"alice/tool": {"to": "alice/tool-ng", "at": iso(NOW)}}) + self.assertEqual(state["renames.json"], + {"alice/tool": {"to": "alice/tool-ng", "at": iso(NOW), "pending": True}}) self.assertEqual(titles[5], "alice/tool-ng") self.assertEqual([c["body"] for c in self.gql(g, "CommentOnEntry")], ["Heads up"]) listed = [entry("alice/tool-ng"), entry("bob/other", issue=2)] on_merge(g, listed, sleep=lambda s: None) self.assertEqual(state["discussions.json"], {"alice/tool-ng": 5, "bob/other": 6, "other/writer": 1}) + self.assertEqual(state["renames.json"], {"alice/tool": {"to": "alice/tool-ng", "at": iso(NOW)}}) self.assertEqual(self.gql(g, "CreateEntryDiscussion"), []) g.calls.clear() on_merge(g, listed, sleep=lambda s: None) self.assertEqual([c for c in g.calls if c[0] == "PUT"], []) - def test_failed_collision_move_is_finished_by_the_next_on_merge(self): + def test_completed_move_is_never_replayed_after_a_relist_and_unlist(self): from scripts.on_merge import run as on_merge - titles = {5: "alice/tool", 6: "bob/kept"} - r, state = store({"discussions.json": {"alice/tool": 5, "bob/kept": 6}}, self.on_merge_routes(titles), + titles = {5: "alice/tool"} + r, state = store({"discussions.json": {"alice/tool": 5}}, self.on_merge_routes(titles), conflicts={"discussions.json": 3}) g = FakeGitHub(r) with self.assertRaises(SystemExit): - finish(g, self.plan([Action("rename", "bob/kept", "", "alice/tool"), - Action("drop", "bob/kept", "Merged", "bob/kept")]), now=NOW) - self.assertEqual(titles, {5: "bob/kept", 6: "Removed: bob/kept"}) - # The stored bob/kept is the dropped entry's thread; the kept entry's Likes are on 5. - on_merge(g, [entry("bob/kept")], sleep=lambda s: None) - self.assertEqual(state["discussions.json"]["bob/kept"], 5) - self.assertNotIn("alice/tool", state["discussions.json"]) - self.assertEqual(self.gql(g, "CreateEntryDiscussion"), []) + finish(g, self.plan([Action("rename", "alice/tool-ng", "", "alice/tool")]), now=NOW) + on_merge(g, [entry("alice/tool-ng")], sleep=lambda s: None) + # Someone lists a new repo under the old name; it gets its own thread. + on_merge(g, [entry("alice/tool-ng"), entry("alice/tool", issue=2)], sleep=lambda s: None) + self.assertEqual(state["discussions.json"]["alice/tool"], 99) + # It comes off the list again while the rename record is still fresh. + on_merge(g, [entry("alice/tool-ng")], sleep=lambda s: None) + self.assertEqual(state["discussions.json"]["alice/tool-ng"], 5) - def test_no_move_onto_a_name_that_is_listed_again(self): - from scripts.on_merge import run as on_merge + def test_completed_rename_record_is_not_replayed(self): + from scripts.on_merge import _follow + self.assertEqual(_follow({"b/x": 5, "a/y": 9}, {}, {"b/x"}), {"b/x": 5, "a/y": 9}) - # alice/tool was renamed away, then someone listed a new repo under the old name. - titles = {5: "alice/tool-ng"} - r, state = store({"discussions.json": {"alice/tool-ng": 5}, - "renames.json": {"alice/tool": {"to": "alice/tool-ng", "at": iso(NOW)}}}, - self.on_merge_routes(titles)) + def test_no_retitle_when_both_rename_writes_fail(self): + titles = {5: "alice/tool"} + r, state = store({"discussions.json": {"alice/tool": 5}}, threads(titles), + conflicts={"discussions.json": 3, "renames.json": 3}) g = FakeGitHub(r) - on_merge(g, [entry("alice/tool-ng"), entry("alice/tool", issue=2)], sleep=lambda s: None) - self.assertEqual(state["discussions.json"], {"alice/tool-ng": 5, "alice/tool": 99}) + with self.assertRaises(SystemExit): + finish(g, self.plan([Action("rename", "alice/tool-ng", "", "alice/tool"), + Action("warn", "alice/tool-ng", "Heads up", "alice/tool")]), now=NOW) + self.assertEqual(titles, {5: "alice/tool"}) + self.assertEqual([c["body"] for c in self.gql(g, "CommentOnEntry")], ["Heads up"]) + + def test_drop_retitled_only_after_its_key_is_gone(self): + titles = {6: "bob/kept"} + r, state = store({"discussions.json": {"bob/kept": 6}}, threads(titles), conflicts={"discussions.json": 3}) + g = FakeGitHub(r) + with self.assertRaises(SystemExit): + finish(g, self.plan([Action("drop", "bob/kept", "Merged", "bob/kept")]), now=NOW) + self.assertEqual(titles, {6: "bob/kept"}) + self.assertEqual([c["body"] for c in self.gql(g, "CommentOnEntry")], ["Merged"]) - def test_collision_keeps_the_older_entrys_thread_through_on_merge(self): - from scripts.devstream.entries import normalize_repo + def test_failed_collision_move_is_finished_by_the_next_on_merge(self): from scripts.on_merge import run as on_merge - from scripts.validate_data import REPO_RE - older = entry("alice/tool", submitted_at="2026-01-01T00:00:00Z", issue=1) - newer = entry("bob/kept", submitted_at="2026-03-01T00:00:00Z", issue=2) - new, actions, rot = plan_rot([older, newer], {"alice/tool": ok("bob/kept")}, NOW, {}) titles = {5: "alice/tool", 6: "bob/kept"} - listing = {"repository": {"discussions": {"nodes": [], "pageInfo": {"hasNextPage": False, "endCursor": None}}}} - - def list_threads(_): - nodes = [{"number": n, "title": t, "author": {"login": "devstreamhq-bot[bot]"}} - for n, t in sorted(titles.items())] - return {"repository": {"discussions": {**listing["repository"]["discussions"], "nodes": nodes}}} - - r, state = store({"discussions.json": {"alice/tool": 5, "bob/kept": 6}}, { - **threads(titles), - "GRAPHQL LikesCategory": {"repository": {"id": "R", "discussionCategories": { - "nodes": [{"id": "C", "slug": "likes"}]}}}, - "GRAPHQL LikesDiscussions": list_threads, - }) + r, state = store({"discussions.json": {"alice/tool": 5, "bob/kept": 6}}, self.on_merge_routes(titles), + conflicts={"discussions.json": 3}) g = FakeGitHub(r) - finish(g, {"changed": True, "actions": [a.__dict__ for a in actions], "rot_state": rot}) - self.assertEqual(titles, {5: "bob/kept", 6: "Removed: bob/kept"}) - self.assertEqual(state["discussions.json"], {"bob/kept": 5}) - dropped = normalize_repo(titles[6]) - self.assertNotIn(dropped, {normalize_repo(e["repo"]) for e in new + [older, newer]}) - self.assertIsNone(REPO_RE.fullmatch(dropped)) - - on_merge(g, new, sleep=lambda s: None) + with self.assertRaises(SystemExit): + finish(g, self.plan([Action("rename", "bob/kept", "", "alice/tool"), + Action("drop", "bob/kept", "Merged", "bob/kept")]), now=NOW) + # The drop's key wasn't removed, so its thread keeps its title until a maintainer looks. + self.assertEqual(titles, {5: "bob/kept", 6: "bob/kept"}) + # The stored bob/kept is the dropped entry's thread; the kept entry's Likes are on 5. + on_merge(g, [entry("bob/kept")], sleep=lambda s: None) self.assertEqual(state["discussions.json"]["bob/kept"], 5) + self.assertNotIn("alice/tool", state["discussions.json"]) self.assertEqual(self.gql(g, "CreateEntryDiscussion"), []) From bb0b56f71ba7fe329044f018b874a6a6dba19dee Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:15:53 -0700 Subject: [PATCH 6/6] fix: stop ssl errors and lost aggregate lists escaping the bots ssl and http.client errors now count as network errors, so they retry and raise as GitHubError like the rest. An adopted aggregate keeps the entries its body already lists. A 404 on a review lookup reads as a deleted issue, like a 410. The test fake's paginate raises on a routed page that isn't a list, as the real client does. --- scripts/devstream/gh.py | 4 +++- scripts/sync_likes.py | 16 +++++++++++----- scripts/tests/fake_gh.py | 18 ++++++++++++------ scripts/tests/test_gh.py | 17 +++++++++++++++++ scripts/tests/test_likes.py | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 12 deletions(-) diff --git a/scripts/devstream/gh.py b/scripts/devstream/gh.py index 743dc40..b9b736b 100644 --- a/scripts/devstream/gh.py +++ b/scripts/devstream/gh.py @@ -1,3 +1,4 @@ +import http.client import json import random import re @@ -15,7 +16,8 @@ TIMEOUT = 30 ATTEMPTS = 3 TRANSIENT = (502, 503, 504) -NETWORK = (urllib.error.URLError, TimeoutError, ConnectionError) +# OSError covers URLError, timeouts, resets and ssl; http.client raises its own on a cut-off response. +NETWORK = (OSError, http.client.HTTPException) # Only a document that opens as a query is a read; anything else (a fragment first, say) may be a mutation. LEADING_COMMENTS = re.compile(r"(\s*#[^\n]*(\n|$))*") READ_QUERY = re.compile(r"\s*(query\b|\{)") diff --git a/scripts/sync_likes.py b/scripts/sync_likes.py index 0b31224..e7c4c9f 100644 --- a/scripts/sync_likes.py +++ b/scripts/sync_likes.py @@ -33,6 +33,7 @@ LABELS = ["report", "needs-review"] BOT = "devstreamhq-bot[bot]" AGGREGATE_TITLE = re.compile(r"Like burst: [0-9]+ entr(y|ies)") +AGGREGATE_LINE = re.compile(r"^- (\S+)$", re.M) # How long a reviewed (closed) issue holds off a new one for the same entry. COOLDOWN = {"burst": DAY, "drip": WEEK} SNAPSHOTS_KEPT = 30 @@ -68,7 +69,7 @@ def _review(gh, flags: dict, now: datetime) -> None: try: issue = gh.get(f"{API}/issues/{n}") except GitHubError as e: - if e.status != 410: # 410: deleted + if e.status not in (404, 410): # deleted # Its record stays pending and is looked up again next hour; the rest still sync. logging.warning("could not look up issue #%s: %s", n, e) continue @@ -149,9 +150,9 @@ def _baseline(gh, now: datetime, renames: dict) -> dict | None: return data -def _open_reports(gh) -> dict[str, int]: +def _open_reports(gh) -> dict[str, dict]: """The bot's open Like issues by title. Only the bot can author these, so a title can't be forged here.""" - return {i["title"]: i["number"] for i in gh.paginate(f"{API}/issues", {"state": "open", "labels": LABELS[0], + return {i["title"]: i for i in gh.paginate(f"{API}/issues", {"state": "open", "labels": LABELS[0], "per_page": 100}) if (i.get("user") or {}).get("login") == BOT and "pull_request" not in i} @@ -162,7 +163,7 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim opened = _open_reports(gh) for kind, key, repo, body in todo[:ISSUE_CAP]: title = f"Like {kind}: {repo}" - n = opened.get(title) or gh.post(f"{API}/issues", {"title": title, "body": body, "labels": LABELS})["number"] + n = (opened.get(title) or {}).get("number") or gh.post(f"{API}/issues", {"title": title, "body": body, "labels": LABELS})["number"] flags.setdefault(key, {})[kind] = _record(kind, n, now) overflow = [(key, kind) for kind, key, _, _ in todo[ISSUE_CAP:]] if not overflow: @@ -170,8 +171,12 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim # A flood of decoys must not become a flood of issues, nor let the target slip its flag. agg = next((r["issue"] for recs in flags.values() for r in recs.values() if r.get("aggregate") and r["reviewed_at"] is None), None) + kept = [] if agg is None: - agg = next((n for t, n in opened.items() if AGGREGATE_TITLE.fullmatch(t)), None) + adopted = next((i for t, i in opened.items() if AGGREGATE_TITLE.fullmatch(t)), None) + if adopted is not None: + # Its records were lost, so the entries it already lists come from its body. + agg, kept = adopted["number"], AGGREGATE_LINE.findall(adopted.get("body") or "") if agg is None: agg = gh.post(f"{API}/issues", {**_aggregate(overflow), "labels": LABELS})["number"] for key, kind in overflow: @@ -181,6 +186,7 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim flags.setdefault(key, {})[kind] = _record(kind, agg, now, aggregate=True) listed = [(k, kind) for k, recs in flags.items() for kind, r in recs.items() if r["issue"] == agg and r["reviewed_at"] is None] + listed += [(k, None) for k in kept if k not in {key for key, _ in listed}] try: state = (gh.patch(f"{API}/issues/{agg}", _aggregate(listed)) or {}).get("state") except GitHubError as e: diff --git a/scripts/tests/fake_gh.py b/scripts/tests/fake_gh.py index 70d65b6..74e3400 100644 --- a/scripts/tests/fake_gh.py +++ b/scripts/tests/fake_gh.py @@ -50,7 +50,8 @@ class FakeGitHub: Only .get returns None on a missing route; post, patch, put and delete raise GitHubError(404). A route whose value is None answers a write like a 204. paginate yields nothing for a missing route (the real client does - the same for a 404 on the first page, unless must_exist; the fake has a single page). + the same for a 404 on the first page, unless must_exist; the fake has a single page), + and raises for a routed page that isn't a list. .last_redirect is reset at the start of every call. @@ -104,15 +105,20 @@ def status(self, path) -> int: def paginate(self, path, params=None, must_exist=False): self.last_redirect = None page = self._find("GET", path, params, None) + # A route answering None is an empty body, which the real client refuses; no route is a 404. + full = path + (("&" if "?" in path else "?") + urllib.parse.urlencode(params) if params else "") + routed = page is not None or any(f"GET {p}" in self.routes for p in (full, path.split("?")[0])) if isinstance(page, Status) and self._ok(page): - page = None - if page is None and must_exist: - raise GitHubError(404, f"page not found: {path}") + page, routed = None, False + if page is None and not routed: + if must_exist: + raise GitHubError(404, f"page not found: {path}") + return if isinstance(page, dict) and "items" in page: page = page["items"] - if page is not None and not isinstance(page, list): + if not isinstance(page, list): raise GitHubError(200, f"not a list page: {path}") - yield from page or [] + yield from page def _write(self, method, path, body): self.last_redirect = None diff --git a/scripts/tests/test_gh.py b/scripts/tests/test_gh.py index 6c712cc..695ad3e 100644 --- a/scripts/tests/test_gh.py +++ b/scripts/tests/test_gh.py @@ -1,5 +1,7 @@ +import http.client import json import socket +import ssl import threading import urllib.error import time @@ -394,6 +396,16 @@ def test_get_network_error_retried(self): self.assertEqual(gh.get("/a"), {"ok": True}) self.assertEqual(len(sent), 2) + def test_ssl_and_http_client_errors_are_network_errors(self): + for err in (ssl.SSLError("bad record"), http.client.IncompleteRead(b"")): + gh, sent, _ = self.gh(err, 200) + self.assertEqual(gh.get("/a"), {"ok": True}) + self.assertEqual(len(sent), 2) + gh, sent, _ = self.gh(err, 200) + with self.assertRaises(GitHubError) as c: + gh.post("/a", {}) + self.assertEqual((c.exception.status, len(sent)), (0, 1)) + def test_three_network_errors_raise(self): gh, sent, _ = self.gh(TimeoutError(), TimeoutError(), TimeoutError(), 200) with self.assertRaises(GitHubError): @@ -524,6 +536,11 @@ def test_last_redirect_reset(self): f.get("/c") self.assertIsNone(f.last_redirect) + def test_paginate_non_list_page_raises(self): + for page in (None, {"a": 1}): + with self.assertRaises(GitHubError): + list(FakeGitHub({"GET /x": page}).paginate("/x")) + def test_paginate_missing_first_page_empty(self): self.assertEqual(list(FakeGitHub({}).paginate("/x")), []) with self.assertRaises(GitHubError): diff --git a/scripts/tests/test_likes.py b/scripts/tests/test_likes.py index d109478..7a3cbad 100644 --- a/scripts/tests/test_likes.py +++ b/scripts/tests/test_likes.py @@ -558,6 +558,40 @@ def test_aggregate_closed_mid_edit_gets_a_new_issue(self): self.assertEqual(w.written("flags.json")["o/s5"]["burst"]["issue"], w.next_issue - 1) self.assertNotEqual(w.next_issue - 1, agg) + def test_review_lookup_404_counts_as_reviewed(self): + w = World([entry("o/a")], {"o/a": [node("x")]}, stats=self.flags()) + + def gone(_): + raise GitHubError(404, "Not Found") + w.routes[f"GET {API}/issues/7"] = gone + w.run() + self.assertEqual(w.written("flags.json")["o/a"]["burst"]["reviewed_at"], iso(NOW)) + + def test_adopted_aggregate_keeps_the_entries_it_listed(self): + first = {f"o/r{j}": burst_nodes(10, 10) for j in range(6)} + later = {f"o/s{j}": [] for j in range(6)} + w = World([entry(r) for r in {**first, **later}], {**first, **later}) + + def boom(_): + raise GitHubError(502, "bad gateway") + w.routes[f"PUT {API}/contents/flags.json"] = boom + with self.assertRaises(GitHubError): + w.run() + agg = w.next_issue - 1 + w.routes[f"GET {API}/issues/{agg}"]["body"] = w.issues_opened()[-1]["body"] + w.persist() + w.routes.pop(f"GET {API}/contents/flags.json") # that PUT never landed + w.routes[f"PUT {API}/contents/flags.json"] = {} + w.set_nodes("o/r5", []) + for r in later: + w.set_nodes(r, burst_nodes(10, 10, reacted=iso(NOW + timedelta(minutes=30)))) + w.run(now=NOW + timedelta(hours=1)) + [(_, path, body)] = w.patches() + self.assertEqual(path, f"{API}/issues/{agg}") + self.assertEqual(body["title"], "Like burst: 7 entries") + self.assertIn("- o/r5", body["body"]) + self.assertIn("- o/s5", body["body"]) + def test_one_failed_review_lookup_skips_only_that_issue(self): stats = {"flags.json": {r: {"burst": {"kind": "burst", "issue": n, "opened_at": iso(NOW), "reviewed_at": None}} for r, n in (("o/a", 7), ("o/b", 8))}}