Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion scripts/devstream/discussions.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@

def list_discussions(gh, category_id: str) -> dict[str, int]:
"""Every thread already in the category, keyed by normalised repo."""
out, cursor = {}, None
out, cursor, seen = {}, None, set()
while True:
page = gh.graphql(_LIST, {"cursor": cursor, "cat": category_id})["repository"]["discussions"]
for n in page["nodes"]:
Expand All @@ -45,6 +45,10 @@ def list_discussions(gh, category_id: str) -> dict[str, int]:
if not page["pageInfo"]["hasNextPage"]:
return out
cursor = page["pageInfo"]["endCursor"]
# A cursor seen before would loop until the job times out.
if not cursor or cursor in seen:
raise GitHubError(200, "discussions cursor repeated")
seen.add(cursor)


def likes_category(gh) -> tuple[str, str]:
Expand Down
32 changes: 26 additions & 6 deletions scripts/devstream/gh.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import http.client
import json
import random
import re
Expand All @@ -6,14 +7,17 @@
import urllib.error
import urllib.parse
import urllib.request
from datetime import timezone
from email.utils import parsedate_to_datetime
from typing import Iterator

MAX_SLEEP = 600
MAX_HOPS = 5
TIMEOUT = 30
ATTEMPTS = 3
TRANSIENT = (502, 503, 504)
NETWORK = (urllib.error.URLError, TimeoutError, ConnectionError)
# OSError covers URLError, timeouts, resets and ssl; http.client raises its own on a cut-off response.
NETWORK = (OSError, http.client.HTTPException)
# Only a document that opens as a query is a read; anything else (a fragment first, say) may be a mutation.
LEADING_COMMENTS = re.compile(r"(\s*#[^\n]*(\n|$))*")
READ_QUERY = re.compile(r"\s*(query\b|\{)")
Expand All @@ -40,6 +44,8 @@ def redirect_request(self, *a, **k):

def _allowed(e: dict, not_found: bool, forbidden: bool) -> bool:
# A FORBIDDEN with no path is the whole query refused, never one hidden repo.
if not isinstance(e, dict):
return False
return (not_found and e.get("type") == "NOT_FOUND") or (forbidden and e.get("type") == "FORBIDDEN" and bool(e.get("path")))


Expand Down Expand Up @@ -79,7 +85,14 @@ def _rate_limit_reset(status, headers) -> int | None:
if headers.get("x-ratelimit-remaining") != "0" and retry is None:
return None
if retry is not None:
return int(time.time()) + int(retry)
if retry.strip().isdigit():
return int(time.time()) + int(retry)
# Retry-After may also be an HTTP date; anything unreadable waits a minute.
try:
at = parsedate_to_datetime(retry)
except (TypeError, ValueError):
return int(time.time()) + 60
return int((at if at.tzinfo else at.replace(tzinfo=timezone.utc)).timestamp())
return int(headers.get("x-ratelimit-reset") or time.time() + 60)

@staticmethod
Expand Down Expand Up @@ -109,7 +122,8 @@ def _request(self, method, path, params=None, body=None, read=None):
except NETWORK as e:
fails += 1
if fails >= ATTEMPTS or not self._retryable(method, read, exc=e):
raise
# As a GitHubError, the callers' per-item error handling applies to it too.
raise GitHubError(0, f"network error: {e}") from e
self._backoff(fails)
continue
if self._retryable(method, read, status) and fails + 1 < ATTEMPTS:
Expand All @@ -128,7 +142,9 @@ def _request(self, method, path, params=None, body=None, read=None):
hops += 1
if hops > MAX_HOPS:
raise GitHubError(status, "too many redirects")
loc = urllib.parse.urlsplit(headers["Location"])
if not headers.get("Location"):
raise GitHubError(status, "redirect without a Location")
loc = urllib.parse.urlsplit(urllib.parse.urljoin(self.base + path, headers["Location"]))
path = loc.path + (f"?{loc.query}" if loc.query else "")
self.last_redirect = path
continue
Expand Down Expand Up @@ -164,7 +180,11 @@ def paginate(self, path, params=None, must_exist=False) -> Iterator[dict]:
raise GitHubError(404, f"page not found: {path}")
first = False
headers, page = r
yield from page["items"] if isinstance(page, dict) and "items" in page else page
items = page["items"] if isinstance(page, dict) and "items" in page else page
if not isinstance(items, list):
# A dict here would yield its keys as if they were items.
raise GitHubError(200, f"not a list page: {path}")
yield from items
m = re.search(r'<([^>]+)>\s*;\s*rel="next"', headers.get("Link") or "")
# Rebuilt against base so the token never leaves the API host.
nxt = urllib.parse.urlsplit(m.group(1)) if m else None
Expand Down Expand Up @@ -201,7 +221,7 @@ def graphql(self, query: str, variables: dict, allow_not_found: bool = False,
raise GitHubError(404, "graphql endpoint not found")
out = r[1] or {}
errors = out.get("errors")
if any(e.get("type") == "RATE_LIMITED" for e in errors or []):
if any(isinstance(e, dict) and e.get("type") == "RATE_LIMITED" for e in errors or []):
raise RateLimited(int(r[0].get("x-ratelimit-reset") or time.time() + 60))
if errors and not all(_allowed(e, allow_not_found, allow_forbidden) for e in errors):
raise GitHubError(200, json.dumps(out["errors"])[:500])
Expand Down
41 changes: 33 additions & 8 deletions scripts/on_merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,19 @@
VISIBLE = {"live", "warned", "flagged"}


def _follow(mapping: dict, pending: dict, listed: set[str]) -> dict:
"""Finish a rename whose key move failed in rot's finish, so the entry keeps its thread and its Likes.

pending holds only renames.json records rot marked pending; a completed move is never replayed.
Overrides a stored new key: that is the dropped entry's thread after a rename collision.
An old name listed again is someone else's entry and keeps its own key.
"""
for old, r in sorted(pending.items(), key=lambda kv: kv[1]["at"]):
if old in mapping and old not in listed:
mapping[normalize_repo(r["to"])] = mapping.pop(old)
return mapping


def run(gh, entries, sleep=time.sleep):
ensure_branch(gh)
cur = read_json(gh, "discussions.json")[0]
Expand All @@ -22,6 +35,11 @@ def run(gh, entries, sleep=time.sleep):
# setdefault: a stored mapping always wins, so a thread retitled onto a listed name can't take it over.
for k, v in list_discussions(gh, category_id).items():
mapping.setdefault(k, v)
# After adopting, so a thread still titled with the old name can't put the old key back.
pending = {k: r for k, r in (read_json(gh, "renames.json")[0] or {}).items()
if isinstance(r, dict) and r.get("pending")}
listed = {normalize_repo(e["repo"]) for e in entries}
_follow(mapping, pending, listed)
todo = {}
for e in entries:
if e["status"] in VISIBLE and normalize_repo(e["repo"]) not in mapping:
Expand All @@ -40,15 +58,22 @@ def run(gh, entries, sleep=time.sleep):
added = {k: v for k, v in mapping.items() if before.get(k) != v}

def merge(stored):
# Re-applied onto a fresh read after a 409; a stored mapping still wins.
return {**added, **{normalize_repo(k): v for k, v in stored.items()}}
if mapping != before:
try:
# Re-applied onto a fresh read after a 409; a stored mapping still wins, once renames are followed.
return {**added, **_follow({normalize_repo(k): v for k, v in stored.items()}, pending, listed)}

def done(stored):
# Replayed once: a later relist and unlist of the old name must never pull its thread back.
return {k: {f: x for f, x in v.items() if f != "pending"} if pending.get(k) == v else v
for k, v in stored.items()}
try:
if mapping != before:
update_json(gh, "discussions.json", merge, "chore(stats): map entry discussions")
except Exception:
if err is None:
raise
logging.exception("could not write discussions.json")
if pending:
update_json(gh, "renames.json", done, "chore(stats): finish renamed repos")
except Exception:
if err is None:
raise
logging.exception("could not write discussions.json or renames.json")


def main():
Expand Down
28 changes: 15 additions & 13 deletions scripts/rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,31 +342,33 @@ def record(data):
# sync_likes replays these over snapshots taken before the rename; past its 7-day baseline they're unused.
kept = {k: v for k, v in data.items() if isinstance(v, dict) and now - _parse(v["at"]) < RENAME_KEEP}
# renames.json keeps one record per old name; a repeat rename replaces it, fine on a weekly schedule.
return {**kept, **{old: {"to": new, "at": _iso(now)} for old, new in moves.items()}}
# pending: the key move failed, so on_merge replays this record once; a completed move is never replayed.
mark = {"pending": True} if "discussions.json" in failed else {}
return {**kept, **{old: {"to": new, "at": _iso(now), **mark} for old, new in moves.items()}}

numbers = dict(read_json(gh, "discussions.json")[0] or {}) # before the move, so dropped entries find their thread
failed = 0
moved = True
failed = set()
for path, mutate, message in (
("rot.json", lambda _: plan["rot_state"], "chore(stats): record failing repos"),
("discussions.json", rekey, "chore(stats): follow renamed repos"),
# After the key move, so record knows whether on_merge has to finish it.
("renames.json", record, "chore(stats): record renamed repos"),
("flags.json", rekey, "chore(stats): follow renamed repos"),
# The site build reads these two by key, so they move now rather than at the next hourly sync.
("likes.json", rekey, "chore(stats): follow renamed repos"),
("meta.json", rekey, "chore(stats): follow renamed repos"),
("renames.json", record, "chore(stats): record renamed repos"),
):
try:
update_json(gh, path, mutate, message)
except Exception:
logging.exception("could not write %s", path)
failed += 1
moved = moved and path != "discussions.json"
# A retitle without its key move would leave the thread named for a key it isn't stored under.
posted, notified = _post(gh, actions, numbers, retitle=moved)
failed += posted + _mark_notified(gh, notified, plan["rot_state"])
if failed:
raise SystemExit(f"{failed} Discussion or stats update(s) failed")
failed.add(path)
# A title must match a key that is stored, or will be once on_merge replays the pending record.
retitle = {"rename": not {"discussions.json", "renames.json"} <= failed, "drop": "discussions.json" not in failed}
posted, notified = _post(gh, actions, numbers, retitle)
total = len(failed) + posted + _mark_notified(gh, notified, plan["rot_state"])
if total:
raise SystemExit(f"{total} Discussion or stats update(s) failed")


def _mark_notified(gh, keys: set[str], planned: dict) -> int:
Expand All @@ -392,7 +394,7 @@ def mark(data):
return 0


def _post(gh, actions, numbers: dict, retitle: bool = True) -> tuple[int, set[str]]:
def _post(gh, actions, numbers: dict, retitle: dict | None = None) -> tuple[int, set[str]]:
"""Comment on, or retitle, each action's Discussion.

numbers maps stats key to Discussion number. Returns (failures, rot keys whose notice is done).
Expand All @@ -408,7 +410,7 @@ def _post(gh, actions, numbers: dict, retitle: bool = True) -> tuple[int, set[st
if d is not None:
if a.kind != "rename":
gh.graphql(_COMMENT, {"discussionId": d["id"], "body": a.message})
if a.kind in ("rename", "drop") and retitle:
if a.kind in ("rename", "drop") and (retitle or {}).get(a.kind, True):
# "Removed: " can never normalise to a listed repo, so on_merge won't adopt the thread.
title = a.repo if a.kind == "rename" else f"{REMOVED_PREFIX}{a.repo}"
gh.graphql(_RETITLE, {"discussionId": d["id"], "title": title})
Expand Down
44 changes: 39 additions & 5 deletions scripts/sync_likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
WEEK = timedelta(days=7)
ISSUE_CAP = 5
LABELS = ["report", "needs-review"]
BOT = "devstreamhq-bot[bot]"
AGGREGATE_TITLE = re.compile(r"Like burst: [0-9]+ entr(y|ies)")
AGGREGATE_LINE = re.compile(r"^- (\S+)$", re.M)
# How long a reviewed (closed) issue holds off a new one for the same entry.
COOLDOWN = {"burst": DAY, "drip": WEEK}
SNAPSHOTS_KEPT = 30
Expand Down Expand Up @@ -66,8 +69,10 @@ def _review(gh, flags: dict, now: datetime) -> None:
try:
issue = gh.get(f"{API}/issues/{n}")
except GitHubError as e:
if e.status != 410: # 410: deleted
raise
if e.status not in (404, 410): # deleted
# Its record stays pending and is looked up again next hour; the rest still sync.
logging.warning("could not look up issue #%s: %s", n, e)
continue
issue = None
if issue is None or issue.get("state") == "closed":
done.add(n)
Expand Down Expand Up @@ -145,17 +150,33 @@ def _baseline(gh, now: datetime, renames: dict) -> dict | None:
return data


def _open_reports(gh) -> dict[str, dict]:
"""The bot's open Like issues by title. Only the bot can author these, so a title can't be forged here."""
return {i["title"]: i for i in gh.paginate(f"{API}/issues", {"state": "open", "labels": LABELS[0],
"per_page": 100})
if (i.get("user") or {}).get("login") == BOT and "pull_request" not in i}


def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetime) -> None:
"""todo is (kind, key, repo, body). Records are set right after each post, so a failure keeps them."""
# A failed flags.json write loses the records of issues already open; adopt those rather than open them again.
opened = _open_reports(gh)
for kind, key, repo, body in todo[:ISSUE_CAP]:
n = gh.post(f"{API}/issues", {"title": f"Like {kind}: {repo}", "body": body, "labels": LABELS})["number"]
title = f"Like {kind}: {repo}"
n = (opened.get(title) or {}).get("number") or gh.post(f"{API}/issues", {"title": title, "body": body, "labels": LABELS})["number"]
flags.setdefault(key, {})[kind] = _record(kind, n, now)
overflow = [(key, kind) for kind, key, _, _ in todo[ISSUE_CAP:]]
if not overflow:
return
# A flood of decoys must not become a flood of issues, nor let the target slip its flag.
agg = next((r["issue"] for recs in flags.values() for r in recs.values()
if r.get("aggregate") and r["reviewed_at"] is None), None)
kept = []
if agg is None:
adopted = next((i for t, i in opened.items() if AGGREGATE_TITLE.fullmatch(t)), None)
if adopted is not None:
# Its records were lost, so the entries it already lists come from its body.
agg, kept = adopted["number"], AGGREGATE_LINE.findall(adopted.get("body") or "")
if agg is None:
agg = gh.post(f"{API}/issues", {**_aggregate(overflow), "labels": LABELS})["number"]
for key, kind in overflow:
Expand All @@ -165,7 +186,20 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim
flags.setdefault(key, {})[kind] = _record(kind, agg, now, aggregate=True)
listed = [(k, kind) for k, recs in flags.items() for kind, r in recs.items()
if r["issue"] == agg and r["reviewed_at"] is None]
gh.patch(f"{API}/issues/{agg}", _aggregate(listed))
listed += [(k, None) for k in kept if k not in {key for key, _ in listed}]
try:
state = (gh.patch(f"{API}/issues/{agg}", _aggregate(listed)) or {}).get("state")
except GitHubError as e:
if e.status not in (404, 410):
# The records already point at it, so the next overflow brings the body up to date.
logging.warning("could not update aggregate issue #%s: %s", agg, e)
return
state = "closed"
if state == "closed":
# Closed or deleted since _review: its review is over, so these entries need an issue of their own.
agg = gh.post(f"{API}/issues", {**_aggregate(overflow), "labels": LABELS})["number"]
for key, kind in overflow:
flags[key][kind] = _record(kind, agg, now, aggregate=True)


def _merge_flags(flags: dict, before: dict):
Expand All @@ -184,7 +218,7 @@ def mutate(cur):


def _rename_in_flight(gh, now, entries, keys, mapping, prev_likes) -> bool:
"""True while a rename has merged but rot's finish hasn't moved discussions.json yet.
"""True while a rename has merged but discussions.json hasn't moved yet (rot's finish, or on_merge within the hour).

The tell: a listed key has no thread, and a key that was ranked last time has a thread but is no
longer listed. An entry removed for good drops out of likes.json, so it can't hold the board.
Expand Down
Loading
Loading