From 4ff4e290201bda562741502e55b2dcbe3dd68bb8 Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:40:07 -0700 Subject: [PATCH 1/9] fix(gh): retry transient 5xx and network errors with backoff GET retries on 500/502/503/504 and network errors; writes retry only on 502/503/504 or a connection that never opened, so a request that may have reached GitHub is not sent twice. Three attempts in all. --- scripts/devstream/gh.py | 35 ++++++++++++++-- scripts/tests/fake_gh.py | 4 +- scripts/tests/test_gh.py | 87 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 5 deletions(-) diff --git a/scripts/devstream/gh.py b/scripts/devstream/gh.py index 786cd64..65af43d 100644 --- a/scripts/devstream/gh.py +++ b/scripts/devstream/gh.py @@ -1,5 +1,7 @@ import json +import random import re +import socket import time import urllib.error import urllib.parse @@ -9,6 +11,9 @@ MAX_SLEEP = 600 MAX_HOPS = 5 TIMEOUT = 30 +ATTEMPTS = 3 +TRANSIENT = (502, 503, 504) +NETWORK = (urllib.error.URLError, TimeoutError, ConnectionError) class GitHubError(Exception): @@ -69,14 +74,38 @@ def _rate_limit_reset(status, headers) -> int | None: return int(time.time()) + int(retry) return int(headers.get("x-ratelimit-reset") or time.time() + 60) + @staticmethod + def _retryable(method, status=None, exc=None) -> bool: + if exc is None: + return status in TRANSIENT or (method == "GET" and status == 500) + if method == "GET": + return True + # A write that may have reached GitHub is never resent: a duplicate issue or comment is worse than a failed run. + return isinstance(getattr(exc, "reason", None), (ConnectionRefusedError, socket.gaierror)) + + def _backoff(self, fails): + self.sleep(2 ** (fails - 1) * (1 + random.random() / 2)) + def _request(self, method, path, params=None, body=None): """Return (headers, parsed json) or None on 404.""" if params: path += ("&" if "?" in path else "?") + urllib.parse.urlencode(params) retried = False hops = 0 + fails = 0 while True: - status, headers, raw = self._send(method, self.base + path, body) + try: + status, headers, raw = self._send(method, self.base + path, body) + except NETWORK as e: + fails += 1 + if fails >= ATTEMPTS or not self._retryable(method, exc=e): + raise + self._backoff(fails) + continue + if self._retryable(method, status) and fails + 1 < ATTEMPTS: + fails += 1 + self._backoff(fails) + continue reset = self._rate_limit_reset(status, headers) if reset is not None: if reset - time.time() > MAX_SLEEP or retried: @@ -147,8 +176,8 @@ def patch(self, path, body): def put(self, path, body): return self._write("PUT", path, body) - def delete(self, path): - return self._write("DELETE", path, None) + def delete(self, path, body=None): + return self._write("DELETE", path, body) def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> dict: """allow_not_found returns data with nulls where only NOT_FOUND errors occurred.""" diff --git a/scripts/tests/fake_gh.py b/scripts/tests/fake_gh.py index a05ccac..be6f2a1 100644 --- a/scripts/tests/fake_gh.py +++ b/scripts/tests/fake_gh.py @@ -118,8 +118,8 @@ def patch(self, path, body): def put(self, path, body): return self._write("PUT", path, body) - def delete(self, path): - return self._write("DELETE", path, None) + def delete(self, path, body=None): + return self._write("DELETE", path, body) def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> dict: self.last_redirect = None diff --git a/scripts/tests/test_gh.py b/scripts/tests/test_gh.py index cf7a2d9..9b582d2 100644 --- a/scripts/tests/test_gh.py +++ b/scripts/tests/test_gh.py @@ -1,5 +1,7 @@ import json +import socket import threading +import urllib.error import time import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -266,6 +268,91 @@ def test_second_rate_limit_raises(self): self.assertEqual(len(self.slept), 1) +class Retries(unittest.TestCase): + """_send is scripted: each item is a status code or an exception to raise.""" + + def gh(self, *script): + slept = [] + gh = GitHub("tok", "http://x", sleep=slept.append) + sent = [] + + def send(method, url, body): + sent.append(method) + r = script[len(sent) - 1] + if isinstance(r, BaseException): + raise r + return r, {}, b'{"ok": true}' if r < 300 else b"{}" + gh._send = send + return gh, sent, slept + + def test_get_502_then_200_succeeds(self): + gh, sent, slept = self.gh(502, 200) + self.assertEqual(gh.get("/a"), {"ok": True}) + self.assertEqual((len(sent), len(slept)), (2, 1)) + + def test_three_503s_raise(self): + gh, sent, slept = self.gh(503, 503, 503, 200) + with self.assertRaises(GitHubError) as c: + gh.get("/a") + self.assertEqual((c.exception.status, len(sent), len(slept)), (503, 3, 2)) + self.assertLess(slept[0], slept[1]) + + def test_404_not_retried(self): + gh, sent, _ = self.gh(404, 200) + self.assertIsNone(gh.get("/a")) + self.assertEqual(len(sent), 1) + + def test_422_not_retried(self): + gh, sent, _ = self.gh(422, 200) + with self.assertRaises(GitHubError): + gh.post("/a", {}) + self.assertEqual(len(sent), 1) + + def test_post_500_not_retried(self): + gh, sent, _ = self.gh(500, 200) + with self.assertRaises(GitHubError): + gh.post("/a", {}) + self.assertEqual(len(sent), 1) + + def test_post_502_retried(self): + gh, sent, _ = self.gh(502, 200) + self.assertEqual(gh.post("/a", {}), {"ok": True}) + self.assertEqual(len(sent), 2) + + def test_get_network_error_retried(self): + for err in (urllib.error.URLError("down"), TimeoutError(), ConnectionResetError()): + gh, sent, _ = self.gh(err, 200) + self.assertEqual(gh.get("/a"), {"ok": True}) + self.assertEqual(len(sent), 2) + + def test_three_network_errors_raise(self): + gh, sent, _ = self.gh(TimeoutError(), TimeoutError(), TimeoutError(), 200) + with self.assertRaises(TimeoutError): + gh.get("/a") + self.assertEqual(len(sent), 3) + + def test_write_retried_only_when_never_sent(self): + refused = urllib.error.URLError(ConnectionRefusedError()) + dns = urllib.error.URLError(socket.gaierror()) + for err in (refused, dns): + gh, sent, _ = self.gh(err, 200) + self.assertEqual(gh.post("/a", {}), {"ok": True}) + self.assertEqual(len(sent), 2) + for err in (TimeoutError(), ConnectionResetError(), urllib.error.URLError("reset")): + gh, sent, _ = self.gh(err, 200) + with self.assertRaises(OSError): + gh.patch("/a", {}) + self.assertEqual(len(sent), 1) + + def test_rate_limit_rules_unchanged(self): + gh, sent, _ = self.gh(200) + gh._send = lambda m, u, b: (sent.append(m), (403, {"x-ratelimit-remaining": "0", + "x-ratelimit-reset": str(LONG)}, b"{}"))[1] + with self.assertRaises(RateLimited): + gh.get("/a") + self.assertEqual(len(sent), 1) + + class FakeTest(unittest.TestCase): def test_same_public_methods(self): pub = lambda c: {n for n in dir(c) if not n.startswith("_") and callable(getattr(c, n))} From 907f40e76386e9c4d1a5aff8ccf2d77796907974 Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:42:20 -0700 Subject: [PATCH 2/9] fix(stats): read files over 1 MB and support compact writes and deletes The contents API returns an empty body with encoding none above 1 MB, so read_json falls back to the git blobs API. write_json can emit compact JSON, and delete_file removes a stats file, retrying once on a 409. --- scripts/devstream/stats.py | 31 +++++++++++++++++++--- scripts/tests/test_stats.py | 52 ++++++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/scripts/devstream/stats.py b/scripts/devstream/stats.py index 6a6b053..998767d 100644 --- a/scripts/devstream/stats.py +++ b/scripts/devstream/stats.py @@ -21,6 +21,12 @@ def read_json(gh, path): cur = gh.get(f"{API}/contents/{path}", {"ref": "stats"}) if cur is None: return None, None + if cur.get("encoding") == "none": + # Over 1 MB the contents API leaves content empty; the blobs API serves up to 100 MB. + blob = gh.get(f"{API}/git/blobs/{cur['sha']}") + if blob is None: + raise GitHubError(404, f"blob for {path} not found") + cur = blob return json.loads(base64.b64decode(cur["content"])), cur["sha"] @@ -34,10 +40,11 @@ def list_dir(gh, path) -> list[str]: return [f["name"] for f in got if f.get("type") == "file"] -def write_json(gh, path, data, sha, message) -> None: +def write_json(gh, path, data, sha, message, compact=False) -> None: + text = json.dumps(data, sort_keys=True, **({"separators": (",", ":")} if compact else {"indent": 2})) body = { "message": message, - "content": base64.b64encode((json.dumps(data, indent=2, sort_keys=True) + "\n").encode()).decode(), + "content": base64.b64encode((text + "\n").encode()).decode(), "branch": "stats", "committer": {"name": BOT, "email": BOT_EMAIL}, } @@ -46,7 +53,7 @@ def write_json(gh, path, data, sha, message) -> None: gh.put(f"{API}/contents/{path}", body) -def update_json(gh, path, mutate, message) -> None: +def update_json(gh, path, mutate, message, compact=False) -> None: """Read, mutate(data) -> new data, write. On a 409 (another writer got there first) re-read and retry once.""" for attempt in (1, 2): data, sha = read_json(gh, path) @@ -54,7 +61,23 @@ def update_json(gh, path, mutate, message) -> None: if new == (data or {}): return try: - write_json(gh, path, new, sha, message) + write_json(gh, path, new, sha, message, compact) + return + except GitHubError as e: + if e.status != 409 or attempt == 2: + raise + + +def delete_file(gh, path, message) -> None: + """Delete a stats-branch file; already gone is fine. Retries once on a 409 like update_json.""" + for attempt in (1, 2): + cur = gh.get(f"{API}/contents/{path}", {"ref": "stats"}) + if cur is None: + return + body = {"message": message, "sha": cur["sha"], "branch": "stats", + "committer": {"name": BOT, "email": BOT_EMAIL}} + try: + gh.delete(f"{API}/contents/{path}", body) return except GitHubError as e: if e.status != 409 or attempt == 2: diff --git a/scripts/tests/test_stats.py b/scripts/tests/test_stats.py index 838eaa4..491c858 100644 --- a/scripts/tests/test_stats.py +++ b/scripts/tests/test_stats.py @@ -1,7 +1,9 @@ +import base64 +import json import unittest from scripts.devstream.gh import GitHubError -from scripts.devstream.stats import API, list_dir +from scripts.devstream.stats import API, delete_file, list_dir, read_json, update_json from scripts.tests.fake_gh import FakeGitHub @@ -19,3 +21,51 @@ def test_non_list_raises(self): gh = FakeGitHub({f"GET {API}/contents/d": {"name": "d", "type": "file", "content": ""}}) with self.assertRaises(GitHubError): list_dir(gh, "d") + + +class ReadJson(unittest.TestCase): + def test_file_over_1mb_is_read_through_the_blob(self): + raw = base64.encodebytes(json.dumps({"a": 1}).encode()).decode() # the blobs API wraps lines + gh = FakeGitHub({ + f"GET {API}/contents/big.json": {"content": "", "encoding": "none", "sha": "s1"}, + f"GET {API}/git/blobs/s1": {"content": raw, "encoding": "base64", "sha": "s1"}, + }) + self.assertEqual(read_json(gh, "big.json"), ({"a": 1}, "s1")) + + def test_missing_blob_raises(self): + gh = FakeGitHub({f"GET {API}/contents/big.json": {"content": "", "encoding": "none", "sha": "s1"}}) + with self.assertRaises(GitHubError): + read_json(gh, "big.json") + + +class Compact(unittest.TestCase): + def test_compact_write_has_no_indent(self): + gh = FakeGitHub({f"PUT {API}/contents/c.json": {}}) + update_json(gh, "c.json", lambda _: {"a": {"b": 1}}, "m", compact=True) + body = base64.b64decode(gh.calls[-1][2]["content"]).decode() + self.assertEqual(body, '{"a":{"b":1}}\n') + + +class DeleteFile(unittest.TestCase): + def test_deletes_with_sha_on_stats_branch(self): + gh = FakeGitHub({f"GET {API}/contents/s/a.json": {"content": "", "sha": "s9"}, + f"DELETE {API}/contents/s/a.json": None}) + delete_file(gh, "s/a.json", "chore: drop") + body = gh.calls[-1][2] + self.assertEqual((body["sha"], body["branch"], body["message"]), ("s9", "stats", "chore: drop")) + self.assertIn("committer", body) + + def test_retries_once_on_409(self): + n = [] + + def conflict(_): + n.append(1) + if len(n) == 1: + raise GitHubError(409, "conflict") + gh = FakeGitHub({f"GET {API}/contents/a.json": {"content": "", "sha": "s9"}, + f"DELETE {API}/contents/a.json": conflict}) + delete_file(gh, "a.json", "m") + self.assertEqual(len(n), 2) + + def test_already_gone_is_fine(self): + delete_file(FakeGitHub({}), "a.json", "m") From 31bbe30f9ac1708c621014f375937cca86ec5f7a Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:42:20 -0700 Subject: [PATCH 3/9] fix(stats): stop stats writes losing data on a 409 and bound file growth Every stats-branch write in sync_likes, on_merge and rot now goes through update_json, which re-reads and re-applies on a 409. flags.json merges this run's records into the stored copy, so an opened issue's number is never lost and never opened twice. The trust cache drops records past the 7-day TTL and is stored compact. Only the last 30 snapshots are kept, which also keeps the directory under the 1,000-entry listing limit. sync_likes skips a run, writing nothing, while a rename is half done: a listed entry has no thread mapping and the mapping holds a key that is no longer listed. --- scripts/on_merge.py | 11 +++-- scripts/rot.py | 7 +-- scripts/sync_likes.py | 65 +++++++++++++++++--------- scripts/tests/test_likes.py | 92 ++++++++++++++++++++++++++++++++++++- 4 files changed, 146 insertions(+), 29 deletions(-) diff --git a/scripts/on_merge.py b/scripts/on_merge.py index 3d32e1c..04b4d81 100644 --- a/scripts/on_merge.py +++ b/scripts/on_merge.py @@ -6,7 +6,7 @@ from scripts.devstream.discussions import ensure_discussion, likes_category, list_discussions from scripts.devstream.gh import GitHub from scripts.devstream.entries import normalize_repo -from scripts.devstream.stats import ensure_branch, read_json, write_json +from scripts.devstream.stats import ensure_branch, read_json, update_json CAP = 40 VISIBLE = {"live", "warned", "flagged"} @@ -14,7 +14,7 @@ def run(gh, entries, sleep=time.sleep): ensure_branch(gh) - cur, sha = read_json(gh, "discussions.json") + cur = read_json(gh, "discussions.json")[0] mapping = {normalize_repo(k): v for k, v in (cur or {}).items()} before = dict(mapping) repo_id, category_id = likes_category(gh) @@ -37,9 +37,14 @@ def run(gh, entries, sleep=time.sleep): raise finally: # Persist what exists even when a create failed, so no Discussion is orphaned. + added = {k: v for k, v in mapping.items() if before.get(k) != v} + + def merge(stored): + # Re-applied onto a fresh read after a 409; a stored mapping still wins. + return {**added, **{normalize_repo(k): v for k, v in stored.items()}} if mapping != before: try: - write_json(gh, "discussions.json", mapping, sha, "chore(stats): map entry discussions") + update_json(gh, "discussions.json", merge, "chore(stats): map entry discussions") except Exception: if err is None: raise diff --git a/scripts/rot.py b/scripts/rot.py index 8cf60ba..4525bca 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -26,7 +26,7 @@ Undecided, check_repo, ) -from scripts.devstream.stats import ensure_branch, read_json, update_json, write_json +from scripts.devstream.stats import ensure_branch, read_json, update_json from scripts.validate_data import validate CHECKED = {"live", "warned", "archived"} @@ -197,7 +197,7 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: entries = load(path) targets = [e for e in select(entries, repo) if e["status"] in CHECKED] ensure_branch(gh) - rot_state, rot_sha = read_json(gh, "rot.json") + rot_state = read_json(gh, "rot.json")[0] rot_state = rot_state or {} results = {} for e in targets: @@ -215,7 +215,7 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: raise SystemExit(1) save(new, path) elif rot != rot_state: - write_json(gh, "rot.json", rot, rot_sha, "chore(stats): record failing repos") + update_json(gh, "rot.json", lambda _: rot, "chore(stats): record failing repos") # Unchanged entries means the only actions are removal notices, safe to post now rot.json is saved. failed, notified = _post(gh, actions, read_json(gh, "discussions.json")[0] or {}) failed += _mark_notified(gh, notified, rot) @@ -267,6 +267,7 @@ def rekey(data): def record(data): # sync_likes replays these over snapshots taken before the rename; past its 7-day baseline they're unused. kept = {k: v for k, v in data.items() if isinstance(v, dict) and now - _parse(v["at"]) < RENAME_KEEP} + # renames.json keeps one record per old name; a repeat rename replaces it, fine on a weekly schedule. return {**kept, **{old: {"to": new, "at": _iso(now)} for old, new in moves.items()}} numbers = dict(read_json(gh, "discussions.json")[0] or {}) # before the move, so dropped entries find their thread diff --git a/scripts/sync_likes.py b/scripts/sync_likes.py index b56edc6..465c579 100644 --- a/scripts/sync_likes.py +++ b/scripts/sync_likes.py @@ -22,7 +22,7 @@ fetch_reactors, rank_entries, ) -from scripts.devstream.stats import API, ensure_branch, list_dir, read_json, write_json +from scripts.devstream.stats import API, delete_file, ensure_branch, list_dir, read_json, update_json from scripts.devstream.trust import TrustCache, ensure_block_access # Flagged entries stay listed and ranked; only Trending hides them. @@ -33,6 +33,7 @@ LABELS = ["report", "needs-review"] # How long a reviewed (closed) issue holds off a new one for the same entry. COOLDOWN = {"burst": DAY, "drip": WEEK} +SNAPSHOTS_KEPT = 30 def _parse(s: str) -> datetime: @@ -165,6 +166,27 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim gh.patch(f"{API}/issues/{agg}", _aggregate(listed)) +def _merge_flags(flags: dict, before: dict): + """A mutate for update_json: lay this run's new and changed records over the stored copy. + + Re-read on a 409, so records another writer stored survive, and so do the issue numbers + this run opened: losing one would open the same issue again next hour. + """ + def mutate(cur): + for k, recs in flags.items(): + for kind, r in recs.items(): + if before.get(k, {}).get(kind) != r: + cur.setdefault(k, {})[kind] = r + return cur + return mutate + + +def _prune_snapshots(gh, today: str) -> None: + names = sorted({n for n in list_dir(gh, "snapshots") if _snapshot_date(n)} | {today}) + for n in names[:-SNAPSHOTS_KEPT]: + delete_file(gh, f"snapshots/{n}", f"chore(stats): drop snapshot {n.removesuffix('.json')}") + + def _under_review(flags: dict, key: str) -> bool: # Closing the review issue is what clears a flag the burst window has outlived. Drips never flag. rec = flags.get(key, {}).get("burst") @@ -180,16 +202,20 @@ def run(gh, entries, now: datetime, excluded: set[str]) -> None: if e["status"] in RANKED: ranked.setdefault(normalize_repo(e["repo"]), e) keys = sorted(ranked) + listed = {normalize_repo(e["repo"]) for e in entries} + if any(k not in mapping for k in keys) and any(k not in listed for k in mapping): + # Rename merged, rot's finish hasn't moved discussions.json yet; the next hourly run picks it up. + logging.warning("a rename is half done; skipping this sync") + return meta = fetch_meta(gh, keys) reactors = fetch_reactors(gh, sorted({mapping[k] for k in keys if k in mapping})) - prev_likes, likes_sha = read_json(gh, "likes.json") - prev_flags, flags_sha = read_json(gh, "flags.json") + prev_likes = read_json(gh, "likes.json")[0] + prev_flags = read_json(gh, "flags.json")[0] flags = copy.deepcopy(prev_flags or {}) _review(gh, flags, now) - cache_data, cache_sha = read_json(gh, "trust-cache.json") - cache = _load_cache(cache_data) + cache = _load_cache(read_json(gh, "trust-cache.json")[0]) ensure_block_access(gh) seen = {} @@ -239,9 +265,8 @@ def trusted(login): "flagged": ranked[k]["status"] == "flagged" or k in bursts or _under_review(flags, k), } - prev_meta, meta_sha = read_json(gh, "meta.json") snap_path = f"snapshots/{now:%Y-%m-%d}.json" - snap_sha = read_json(gh, snap_path)[1] + flags_write = ("flags.json", _merge_flags(flags, prev_flags or {}), "chore(stats): record Like reviews") if candidates: try: @@ -250,25 +275,21 @@ def trusted(login): # Without the records of issues already opened, the next run would open them again. if flags != (prev_flags or {}): try: - write_json(gh, "flags.json", flags, flags_sha, "chore(stats): record Like reviews") + update_json(gh, *flags_write) except Exception: logging.exception("could not write flags.json") raise - writes = [ - # First, so a later failed write can't lose the record of issues already opened. - ("flags.json", flags, prev_flags or {}, flags_sha, "chore(stats): record Like reviews"), - ("meta.json", meta, prev_meta, meta_sha, "chore(stats): update repo meta"), - ("likes.json", out, prev_likes, likes_sha, "chore(stats): update likes"), - ] - if snap_sha is None: - snap = {k: {"likes": v["likes"], "rank": v["rank"]} for k, v in out.items()} - writes.append((snap_path, snap, None, None, f"chore(stats): snapshot likes for {now:%Y-%m-%d}")) - writes.append(("trust-cache.json", cache.entries, cache_data, cache_sha, "chore(stats): update trust cache")) - for path, data, before, sha, message in writes: - if data != before: - write_json(gh, path, data, sha, message) - + snap = {k: {"likes": v["likes"], "rank": v["rank"]} for k, v in out.items()} + # Each is re-applied onto a fresh read if another writer 409s it first. + update_json(gh, *flags_write) # first, so a later failed write can't lose the record of issues opened + update_json(gh, "meta.json", lambda _: meta, "chore(stats): update repo meta") + update_json(gh, "likes.json", lambda _: out, "chore(stats): update likes") + update_json(gh, snap_path, lambda cur: cur or snap, f"chore(stats): snapshot likes for {now:%Y-%m-%d}") + # Past the TTL a record is never read again, and unpruned the file passes 1 MB. + fresh = {k: v for k, v in cache.entries.items() if cache.get(k, now) is not None} + update_json(gh, "trust-cache.json", lambda _: fresh, "chore(stats): update trust cache", compact=True) + _prune_snapshots(gh, snap_path.removeprefix("snapshots/")) def main(): excluded = {u.strip().lower() for u in os.environ.get("EXCLUDED_USERS", "").split(",") if u.strip()} diff --git a/scripts/tests/test_likes.py b/scripts/tests/test_likes.py index 7d006b2..8acac7c 100644 --- a/scripts/tests/test_likes.py +++ b/scripts/tests/test_likes.py @@ -65,7 +65,8 @@ def __init__(self, entries, discussions, users=None, stats=None, stars=None, iss "content": base64.b64encode(json.dumps(data).encode()).decode(), "sha": f"sha-{path}"} snaps = sorted(p for p in self.stats if p.startswith("snapshots/")) if snaps: - routes[f"GET {API}/contents/snapshots"] = [{"name": p.split("/")[1], "type": "file"} for p in snaps] + routes[f"GET {API}/contents/snapshots"] = [ + {"name": p.split("/")[1], "type": "file", "sha": f"sha-{p}"} for p in snaps] days = [f"snapshots/{NOW + timedelta(days=d):%Y-%m-%d}.json" for d in range(12)] for path in ("meta.json", "likes.json", "flags.json", "trust-cache.json", *days): routes[f"PUT {API}/contents/{path}"] = {} @@ -755,5 +756,94 @@ def test_missing_stats_branch(self): self.assertEqual(w.puts(), []) +class Conflicts(unittest.TestCase): + """A 409 on the stats branch means another writer got there first: re-read, re-apply, write again.""" + + def conflict_once(self, w, path, then=None): + n = [] + + def put(_): + n.append(1) + if len(n) == 1: + if then: + then() + raise GitHubError(409, "conflict") + return {} + w.routes[f"PUT {API}/contents/{path}"] = put + return n + + def test_flags_409_then_success_keeps_both_records(self): + w = World([entry("o/a")], {"o/a": burst_nodes(10, 10)}) + other = {"o/z": {"burst": {"kind": "burst", "issue": 5, "opened_at": iso(NOW), "reviewed_at": None}}} + + def someone_wrote(): + w.routes[f"GET {API}/contents/flags.json"] = { + "content": base64.b64encode(json.dumps(other).encode()).decode(), "sha": "sha-new"} + n = self.conflict_once(w, "flags.json", someone_wrote) + w.run() + self.assertEqual(len(n), 2) + self.assertEqual(titles(w), ["Like burst: o/a"]) + got = w.written("flags.json") + self.assertEqual((got["o/a"]["burst"]["issue"], got["o/z"]["burst"]["issue"]), (100, 5)) + self.assertEqual(w.puts()[-5][2]["sha"], "sha-new") + + def test_flags_409_then_success_opens_no_duplicate_next_run(self): + w = World([entry("o/a")], {"o/a": burst_nodes(10, 10)}) + self.conflict_once(w, "flags.json") + w.run() + w.persist() + w.run(now=NOW + timedelta(minutes=5)) + self.assertEqual(w.issues_opened(), []) + + def test_likes_409_then_success(self): + w = World([entry("o/a")], {"o/a": [node("x"), node("y")]}) + n = self.conflict_once(w, "likes.json") + w.run() + self.assertEqual(len(n), 2) + self.assertEqual(w.written("likes.json")["o/a"]["likes"], 2) + + +class Size(unittest.TestCase): + def test_trust_cache_pruned_past_ttl_and_compact(self): + stale = {"checked_at": iso(NOW - timedelta(days=8)), "trusted": True, "reason": None, "created_at": OLD} + w = World([entry("o/a")], {"o/a": [node("x")]}, stats={"trust-cache.json": {"gone": stale}}) + w.run() + self.assertEqual(list(w.written("trust-cache.json")), ["x"]) + put = [b for _, p, b in w.puts() if p.endswith("trust-cache.json")][0] + self.assertNotIn("\n ", base64.b64decode(put["content"]).decode()) + + def test_only_the_last_30_snapshots_are_kept(self): + stats = {f"snapshots/{NOW - timedelta(days=d):%Y-%m-%d}.json": {} for d in range(1, 33)} + w = World([entry("o/a")], {"o/a": [node("x")]}, stats=stats) + for p in stats: + w.routes[f"DELETE {API}/contents/{p}"] = None + w.run() + deletes = [(i, c) for i, c in enumerate(w.gh.calls) if c[0] == "DELETE"] + old = [f"{API}/contents/snapshots/{NOW - timedelta(days=d):%Y-%m-%d}.json" for d in (32, 31, 30)] + self.assertEqual([c[1] for _, c in deletes], old) + self.assertEqual(deletes[0][1][2]["sha"], f"sha-snapshots/{NOW - timedelta(days=32):%Y-%m-%d}.json") + snap_put = max(i for i, c in enumerate(w.gh.calls) if c[0] == "PUT" and c[1].endswith(f"{TODAY}.json")) + self.assertGreater(deletes[0][0], snap_put) + + def test_30_or_fewer_snapshots_deletes_nothing(self): + stats = {f"snapshots/{NOW - timedelta(days=d):%Y-%m-%d}.json": {} for d in range(1, 30)} + w = World([entry("o/a")], {"o/a": [node("x")]}, stats=stats) + w.run() + self.assertFalse([c for c in w.gh.calls if c[0] == "DELETE"]) + + +class HalfDoneRename(unittest.TestCase): + def test_listed_key_unmapped_while_unlisted_key_mapped_writes_nothing(self): + # The rename PR merged but rot's finish hasn't moved discussions.json yet. + w = World([entry("o/new")], {"o/old": [node("x"), node("y")]}) + w.run() + self.assertFalse([c for c in w.gh.calls if c[0] in ("PUT", "POST", "PATCH", "DELETE")]) + + def test_new_entry_without_thread_still_syncs(self): + w = World([entry("o/a"), entry("o/b")], {"o/a": [node("x")]}) + w.run() + self.assertEqual(w.written("likes.json")["o/b"]["likes"], 0) + + if __name__ == "__main__": unittest.main() From 941f309fcad0b5b0295f036406ea04636676236b Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:48:04 -0700 Subject: [PATCH 4/9] test(site): cover the Trending age gate with a future-dated entry The new fixture row climbs by 9 with no seed flag, so only the 7-day age gate keeps its data-trend mark off. Counts and orderings that the extra row shifts are updated. --- site/tests/data.test.mjs | 8 ++++++-- site/tests/dist.test.mjs | 10 +++++++++- site/tests/embeds.test.mjs | 2 +- site/tests/fixtures/entries.json | 12 ++++++++++++ site/tests/fixtures/stats/likes.json | 7 +++++++ 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/site/tests/data.test.mjs b/site/tests/data.test.mjs index e75306b..6b1ab8b 100644 --- a/site/tests/data.test.mjs +++ b/site/tests/data.test.mjs @@ -53,6 +53,7 @@ test("sort by likes then stars then name", () => { "quiet-co/tiny-queue", "Pixel-Forge/shader-kit", "burst/likes-spike", + "future/fresh-entry", "evil/hostile-stats", "fresh/no-stats-yet", ]); @@ -78,7 +79,10 @@ test("archived and superseded appear only in the archive list", () => { }); test("new sorts by submission time, newest first", () => { - assert.equal(newest(all().entries)[0].repo, "fresh/no-stats-yet"); + assert.deepEqual(names(newest(all().entries)).slice(0, 2), [ + "future/fresh-entry", + "fresh/no-stats-yet", + ]); }); test("stats are merged by lowercased repo key", () => { @@ -110,7 +114,7 @@ test("missing stats files are treated as empty", () => { dataDir: fixtures, statsDir: `${fixtures}/does-not-exist`, }); - assert.equal(liveList(entries).length, 7); + assert.equal(liveList(entries).length, 8); assert.ok(entries.every((e) => e.likes === 0 && e.rank === null)); }); diff --git a/site/tests/dist.test.mjs b/site/tests/dist.test.mjs index e42fb9b..4edf847 100644 --- a/site/tests/dist.test.mjs +++ b/site/tests/dist.test.mjs @@ -105,7 +105,7 @@ test("top10.svg and badge.json are built", () => { assert.ok( svg.startsWith(" { @@ -139,6 +139,14 @@ test("the Trending tab marks exactly the rows trending() allows", async () => { assert.ok(rows.length > 0, page); // zz-tie/same-likes climbs by 3, under TRENDING_MIN_DELTA, so the hero would skip it. assert.ok(!marked.includes("zz-tie/same-likes"), page); + // future/fresh-entry climbs by 9 but isn't 7 days old and has no seed flag: the age gate. + if (page === "index.html") { + assert.ok( + rows.some((r) => r.includes('data-q="future/fresh-entry ')), + page, + ); + } + assert.ok(!marked.includes("future/fresh-entry"), page); assert.deepEqual( marked.sort(), want.filter((r) => rows.some((x) => x.includes(`data-q="${r} `))).sort(), diff --git a/site/tests/embeds.test.mjs b/site/tests/embeds.test.mjs index d2be268..f0d3662 100644 --- a/site/tests/embeds.test.mjs +++ b/site/tests/embeds.test.mjs @@ -54,7 +54,7 @@ test("badge.json has the shields.io endpoint shape", () => { assert.deepEqual(badge(live), { schemaVersion: 1, label: "listed", - message: "7 repos", + message: "8 repos", color: BRAND_HEX, }); assert.equal(badge(live.slice(0, 1)).message, "1 repo"); diff --git a/site/tests/fixtures/entries.json b/site/tests/fixtures/entries.json index f92521f..7159d42 100644 --- a/site/tests/fixtures/entries.json +++ b/site/tests/fixtures/entries.json @@ -120,6 +120,18 @@ "warned_at": null, "superseded_by": null, "added_by_seed": false + }, + { + "repo": "future/fresh-entry", + "category": "testing", + "description": "Climbing fast but too new for Trending.", + "submitted_by": "fay", + "submitted_at": "2999-01-01T00:00:00Z", + "issue": 11, + "status": "live", + "warned_at": null, + "superseded_by": null, + "added_by_seed": false } ] } diff --git a/site/tests/fixtures/stats/likes.json b/site/tests/fixtures/stats/likes.json index 4ff4725..259eab5 100644 --- a/site/tests/fixtures/stats/likes.json +++ b/site/tests/fixtures/stats/likes.json @@ -47,5 +47,12 @@ "rank": "