diff --git a/.github/workflows/auto-approve.yml b/.github/workflows/auto-approve.yml index 5f96f37..e87f116 100644 --- a/.github/workflows/auto-approve.yml +++ b/.github/workflows/auto-approve.yml @@ -15,6 +15,7 @@ jobs: github.event.pull_request.user.login == 'devstreamhq-bot[bot]' && github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read pull-requests: write diff --git a/.github/workflows/cflite_pr.yml b/.github/workflows/cflite_pr.yml index 1899c20..4c3b656 100644 --- a/.github/workflows/cflite_pr.yml +++ b/.github/workflows/cflite_pr.yml @@ -10,6 +10,7 @@ permissions: {} jobs: PR: runs-on: ubuntu-latest + timeout-minutes: 20 permissions: contents: read concurrency: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index f6f1892..57328b0 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -15,6 +15,7 @@ jobs: # analyze (javascript-typescript) and analyze (actions). analyze: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: actions: read contents: read diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b1f08db..c858b29 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -28,6 +28,7 @@ jobs: build: if: github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main') runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read env: @@ -69,6 +70,7 @@ jobs: deploy: needs: build runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read steps: diff --git a/.github/workflows/likes.yml b/.github/workflows/likes.yml index 5405001..40fafa1 100644 --- a/.github/workflows/likes.yml +++ b/.github/workflows/likes.yml @@ -16,6 +16,7 @@ concurrency: jobs: likes: runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read steps: diff --git a/.github/workflows/on-merge.yml b/.github/workflows/on-merge.yml index 9473c08..22accb1 100644 --- a/.github/workflows/on-merge.yml +++ b/.github/workflows/on-merge.yml @@ -21,6 +21,7 @@ concurrency: jobs: discussions: runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read steps: diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 81dae54..e1fb1bd 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -9,6 +9,7 @@ permissions: {} jobs: validate: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read steps: @@ -25,6 +26,7 @@ jobs: test: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read steps: @@ -41,6 +43,7 @@ jobs: lint: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e9be67..b5727a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,6 +9,7 @@ permissions: {} jobs: release: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: write id-token: write diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml index 84ceadb..bc82b6b 100644 --- a/.github/workflows/reports.yml +++ b/.github/workflows/reports.yml @@ -22,6 +22,7 @@ jobs: github.event.issue.user.type != 'Bot' && github.event.issue.user.login != 'devstreamhq-bot[bot]') runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read outputs: @@ -96,6 +97,7 @@ jobs: group: list-writes cancel-in-progress: false runs-on: ubuntu-latest + timeout-minutes: 35 permissions: contents: read steps: @@ -188,16 +190,39 @@ jobs: [ "$(gh pr view "$PR_URL" --json state --jq .state)" = MERGED ] && exit 0 echo "::error::$PR_URL did not merge within 15 minutes and is now closed" exit 1 + # Minted after the merge wait, so no later step runs on a token that expired during it. + - id: late_token + if: always() && steps.pr.outputs.url != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + - id: late_issues_token + if: always() && steps.issues_token.outputs.token != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-issues: write # Only now is "hidden" true, so the comment waits for the merge. - env: - GITHUB_TOKEN: ${{ steps.issues_token.outputs.token }} + GITHUB_TOKEN: ${{ steps.late_issues_token.outputs.token || steps.issues_token.outputs.token }} RESULT: ${{ runner.temp }}/result/result.json run: python -m scripts.reports issue --result "$RESULT" - # A genuine failure (apply error, push, PR, merge timeout) is parked for a maintainer: + # A cancel or job timeout releases the lock, so the armed PR must not merge behind the next writer. + # Timeouts reach this step too: the runner re-evaluates later steps' if: on any job cancellation. + - if: cancelled() && steps.pr.outputs.url != '' + env: + GH_TOKEN: ${{ steps.late_token.outputs.token || steps.token.outputs.token }} + PR_URL: ${{ steps.pr.outputs.url }} + run: gh pr close "$PR_URL" || true # fails if it merged meanwhile + # A genuine failure (apply error, push, PR, merge timeout, cancel) is parked for a maintainer: # needs-review stops the sweeper retrying it. The issue stays open. - - if: failure() && steps.issues_token.outputs.token != '' && steps.meta.outputs.issue != '' + - if: (failure() || cancelled()) && steps.issues_token.outputs.token != '' && steps.meta.outputs.issue != '' env: - GITHUB_TOKEN: ${{ steps.issues_token.outputs.token }} + GITHUB_TOKEN: ${{ steps.late_issues_token.outputs.token || steps.issues_token.outputs.token }} RESULT: ${{ runner.temp }}/result/result.json run: python -m scripts.reports issue --result "$RESULT" --error @@ -205,6 +230,7 @@ jobs: needs: decide if: needs.decide.outputs.has == 'true' && needs.decide.outputs.hide != 'true' runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read steps: diff --git a/.github/workflows/rot.yml b/.github/workflows/rot.yml index de4c782..7511d20 100644 --- a/.github/workflows/rot.yml +++ b/.github/workflows/rot.yml @@ -25,6 +25,7 @@ concurrency: jobs: rot: runs-on: ubuntu-latest + timeout-minutes: 120 permissions: contents: read steps: @@ -88,12 +89,29 @@ jobs: [ "$(gh pr view "$PR_URL" --json state --jq .state)" = MERGED ] && exit 0 echo "::error::$PR_URL did not merge within 15 minutes and is now closed" exit 1 + # Minted after the merge wait, so no later step runs on a token that expired during it. + - id: late_token + if: always() && steps.pr.outputs.url != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + permission-discussions: write - if: steps.plan.outputs.changed == 'true' env: - GITHUB_TOKEN: ${{ steps.token.outputs.token }} + GITHUB_TOKEN: ${{ steps.late_token.outputs.token || steps.token.outputs.token }} run: python -m scripts.rot finish --plan "$RUNNER_TEMP/plan.json" # Only after finish has moved a renamed entry's thread to its new key, or this would open a second one. - if: steps.plan.outputs.changed == 'true' env: - GITHUB_TOKEN: ${{ steps.token.outputs.token }} + GITHUB_TOKEN: ${{ steps.late_token.outputs.token || steps.token.outputs.token }} run: python -m scripts.on_merge + # A cancel or job timeout releases the lock, so the armed PR must not merge behind the next writer. + # Timeouts reach this step too: the runner re-evaluates later steps' if: on any job cancellation. + - if: cancelled() && steps.pr.outputs.url != '' + env: + GH_TOKEN: ${{ steps.late_token.outputs.token || steps.token.outputs.token }} + PR_URL: ${{ steps.pr.outputs.url }} + run: gh pr close "$PR_URL" || true # fails if it merged meanwhile diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index e25d3ea..626e86a 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -15,6 +15,7 @@ jobs: # containers, and only its approved actions as steps. analysis: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: # Upload the results to code scanning. security-events: write diff --git a/.github/workflows/stars.yml b/.github/workflows/stars.yml index 45d31bc..f51d1d2 100644 --- a/.github/workflows/stars.yml +++ b/.github/workflows/stars.yml @@ -11,6 +11,7 @@ permissions: {} jobs: stars: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read steps: diff --git a/.github/workflows/submit.yml b/.github/workflows/submit.yml index bc676ad..8fa96cb 100644 --- a/.github/workflows/submit.yml +++ b/.github/workflows/submit.yml @@ -22,6 +22,7 @@ jobs: decide: if: github.event_name == 'workflow_dispatch' || contains(github.event.issue.labels.*.name, 'submission') runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read outputs: @@ -71,6 +72,7 @@ jobs: needs: decide if: needs.decide.outputs.ok == 'true' runs-on: ubuntu-latest + timeout-minutes: 35 permissions: contents: read steps: @@ -162,9 +164,25 @@ jobs: [ "$(gh pr view "$PR_URL" --json state --jq .state)" = MERGED ] && exit 0 echo "::error::$PR_URL did not merge within 15 minutes and is now closed" exit 1 + # Minted after the merge wait, so no later step runs on a token that expired during it. + - id: late_token + if: always() && steps.pr.outputs.url != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + - id: late_issues_token + if: always() && steps.issues_token.outputs.token != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-issues: write - if: steps.apply.outputs.branch != '' env: - GH_TOKEN: ${{ steps.issues_token.outputs.token }} + GH_TOKEN: ${{ steps.late_issues_token.outputs.token || steps.issues_token.outputs.token }} RESULT: ${{ runner.temp }}/result/result.json ISSUE: ${{ steps.meta.outputs.issue }} run: | @@ -189,7 +207,7 @@ jobs: run: python -m scripts.on_merge - if: steps.apply.outputs.branch == '' env: - GH_TOKEN: ${{ steps.issues_token.outputs.token }} + GH_TOKEN: ${{ steps.late_issues_token.outputs.token || steps.issues_token.outputs.token }} RESULT: ${{ runner.temp }}/result/result.json ISSUE: ${{ steps.meta.outputs.issue }} run: | @@ -197,11 +215,18 @@ jobs: gh issue comment "$ISSUE" --body-file "$RUNNER_TEMP/comment.md" gh issue edit "$ISSUE" --add-label rejected gh issue close "$ISSUE" --reason "not planned" - # A genuine failure (apply error, push, PR, merge timeout) is parked for a human: + # A cancel or job timeout releases the lock, so the armed PR must not merge behind the next writer. + # Timeouts reach this step too: the runner re-evaluates later steps' if: on any job cancellation. + - if: cancelled() && steps.pr.outputs.url != '' + env: + GH_TOKEN: ${{ steps.late_token.outputs.token || steps.token.outputs.token }} + PR_URL: ${{ steps.pr.outputs.url }} + run: gh pr close "$PR_URL" || true # fails if it merged meanwhile + # A genuine failure (apply error, push, PR, merge timeout, cancel) is parked for a human: # the sweeper skips needs-review, so it can't retry forever. The issue stays open. - - if: failure() && steps.issues_token.outputs.token != '' && steps.meta.outputs.issue != '' + - if: (failure() || cancelled()) && steps.issues_token.outputs.token != '' && steps.meta.outputs.issue != '' env: - GH_TOKEN: ${{ steps.issues_token.outputs.token }} + GH_TOKEN: ${{ steps.late_issues_token.outputs.token || steps.issues_token.outputs.token }} RESULT: ${{ runner.temp }}/result/result.json ISSUE: ${{ steps.meta.outputs.issue }} run: | @@ -213,6 +238,7 @@ jobs: needs: decide if: needs.decide.outputs.ok == 'false' runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read steps: diff --git a/.github/workflows/sweep.yml b/.github/workflows/sweep.yml index bb72f2b..8a37a7a 100644 --- a/.github/workflows/sweep.yml +++ b/.github/workflows/sweep.yml @@ -18,6 +18,7 @@ concurrency: jobs: sweep: runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read steps: diff --git a/scripts/devstream/gh.py b/scripts/devstream/gh.py index 786cd64..fbcc3c7 100644 --- a/scripts/devstream/gh.py +++ b/scripts/devstream/gh.py @@ -1,5 +1,7 @@ import json +import random import re +import socket import time import urllib.error import urllib.parse @@ -9,6 +11,9 @@ MAX_SLEEP = 600 MAX_HOPS = 5 TIMEOUT = 30 +ATTEMPTS = 3 +TRANSIENT = (502, 503, 504) +NETWORK = (urllib.error.URLError, TimeoutError, ConnectionError) class GitHubError(Exception): @@ -69,14 +74,40 @@ def _rate_limit_reset(status, headers) -> int | None: return int(time.time()) + int(retry) return int(headers.get("x-ratelimit-reset") or time.time() + 60) - def _request(self, method, path, params=None, body=None): + @staticmethod + def _retryable(method, read, status=None, exc=None) -> bool: + """read: a GET or a GraphQL query. PUT, PATCH and DELETE give the same result twice (contents PUTs are sha-guarded).""" + if exc is None: + return (method != "POST" or read) and (status in TRANSIENT or (read and status == 500)) + if read: + return True + # Anything else may have reached GitHub; a POST or mutation sent twice duplicates an issue, thread or comment. + return isinstance(getattr(exc, "reason", None), (ConnectionRefusedError, socket.gaierror)) + + def _backoff(self, fails): + self.sleep(2 ** (fails - 1) * (1 + random.random() / 2)) + + def _request(self, method, path, params=None, body=None, read=None): """Return (headers, parsed json) or None on 404.""" + read = method == "GET" if read is None else read if params: path += ("&" if "?" in path else "?") + urllib.parse.urlencode(params) retried = False hops = 0 + fails = 0 while True: - status, headers, raw = self._send(method, self.base + path, body) + try: + status, headers, raw = self._send(method, self.base + path, body) + except NETWORK as e: + fails += 1 + if fails >= ATTEMPTS or not self._retryable(method, read, exc=e): + raise + self._backoff(fails) + continue + if self._retryable(method, read, status) and fails + 1 < ATTEMPTS: + fails += 1 + self._backoff(fails) + continue reset = self._rate_limit_reset(status, headers) if reset is not None: if reset - time.time() > MAX_SLEEP or retried: @@ -147,13 +178,14 @@ def patch(self, path, body): def put(self, path, body): return self._write("PUT", path, body) - def delete(self, path): - return self._write("DELETE", path, None) + def delete(self, path, body=None): + return self._write("DELETE", path, body) def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> dict: """allow_not_found returns data with nulls where only NOT_FOUND errors occurred.""" self.last_redirect = None - r = self._request("POST", "/graphql", body={"query": query, "variables": variables}) + r = self._request("POST", "/graphql", body={"query": query, "variables": variables}, + read=not query.lstrip().startswith("mutation")) if r is None: raise GitHubError(404, "graphql endpoint not found") out = r[1] or {} diff --git a/scripts/devstream/stats.py b/scripts/devstream/stats.py index 6a6b053..d41a6ba 100644 --- a/scripts/devstream/stats.py +++ b/scripts/devstream/stats.py @@ -9,6 +9,7 @@ API = "/repos/devstreamhq/devstream" BOT = "devstreamhq-bot[bot]" BOT_EMAIL = "353074438+devstreamhq-bot[bot]@users.noreply.github.com" +ATTEMPTS = 3 def ensure_branch(gh) -> None: @@ -21,6 +22,12 @@ def read_json(gh, path): cur = gh.get(f"{API}/contents/{path}", {"ref": "stats"}) if cur is None: return None, None + if cur.get("encoding") == "none": + # Over 1 MB the contents API leaves content empty; the blobs API serves up to 100 MB. + blob = gh.get(f"{API}/git/blobs/{cur['sha']}") + if blob is None: + raise GitHubError(404, f"blob for {path} not found") + cur = blob return json.loads(base64.b64decode(cur["content"])), cur["sha"] @@ -34,10 +41,11 @@ def list_dir(gh, path) -> list[str]: return [f["name"] for f in got if f.get("type") == "file"] -def write_json(gh, path, data, sha, message) -> None: +def write_json(gh, path, data, sha, message, compact=False) -> None: + text = json.dumps(data, sort_keys=True, **({"separators": (",", ":")} if compact else {"indent": 2})) body = { "message": message, - "content": base64.b64encode((json.dumps(data, indent=2, sort_keys=True) + "\n").encode()).decode(), + "content": base64.b64encode((text + "\n").encode()).decode(), "branch": "stats", "committer": {"name": BOT, "email": BOT_EMAIL}, } @@ -46,16 +54,34 @@ def write_json(gh, path, data, sha, message) -> None: gh.put(f"{API}/contents/{path}", body) -def update_json(gh, path, mutate, message) -> None: - """Read, mutate(data) -> new data, write. On a 409 (another writer got there first) re-read and retry once.""" - for attempt in (1, 2): +def update_json(gh, path, mutate, message, compact=False) -> None: + """Read, mutate(data) -> new data, write. On a 409 (another writer got there first) re-read and retry, 3 attempts in all.""" + for attempt in range(1, ATTEMPTS + 1): data, sha = read_json(gh, path) new = mutate(copy.deepcopy(data) if data is not None else {}) if new == (data or {}): return try: - write_json(gh, path, new, sha, message) + write_json(gh, path, new, sha, message, compact) return except GitHubError as e: - if e.status != 409 or attempt == 2: + if e.status != 409 or attempt == ATTEMPTS: + raise + + +def delete_file(gh, path, message) -> None: + """Delete a stats-branch file; already gone is fine. Retries a 409 like update_json.""" + for attempt in range(1, ATTEMPTS + 1): + cur = gh.get(f"{API}/contents/{path}", {"ref": "stats"}) + if cur is None: + return + body = {"message": message, "sha": cur["sha"], "branch": "stats", + "committer": {"name": BOT, "email": BOT_EMAIL}} + try: + gh.delete(f"{API}/contents/{path}", body) + return + except GitHubError as e: + if e.status == 404: # another run deleted it first + return + if e.status != 409 or attempt == ATTEMPTS: raise diff --git a/scripts/on_merge.py b/scripts/on_merge.py index 3d32e1c..04b4d81 100644 --- a/scripts/on_merge.py +++ b/scripts/on_merge.py @@ -6,7 +6,7 @@ from scripts.devstream.discussions import ensure_discussion, likes_category, list_discussions from scripts.devstream.gh import GitHub from scripts.devstream.entries import normalize_repo -from scripts.devstream.stats import ensure_branch, read_json, write_json +from scripts.devstream.stats import ensure_branch, read_json, update_json CAP = 40 VISIBLE = {"live", "warned", "flagged"} @@ -14,7 +14,7 @@ def run(gh, entries, sleep=time.sleep): ensure_branch(gh) - cur, sha = read_json(gh, "discussions.json") + cur = read_json(gh, "discussions.json")[0] mapping = {normalize_repo(k): v for k, v in (cur or {}).items()} before = dict(mapping) repo_id, category_id = likes_category(gh) @@ -37,9 +37,14 @@ def run(gh, entries, sleep=time.sleep): raise finally: # Persist what exists even when a create failed, so no Discussion is orphaned. + added = {k: v for k, v in mapping.items() if before.get(k) != v} + + def merge(stored): + # Re-applied onto a fresh read after a 409; a stored mapping still wins. + return {**added, **{normalize_repo(k): v for k, v in stored.items()}} if mapping != before: try: - write_json(gh, "discussions.json", mapping, sha, "chore(stats): map entry discussions") + update_json(gh, "discussions.json", merge, "chore(stats): map entry discussions") except Exception: if err is None: raise diff --git a/scripts/rot.py b/scripts/rot.py index 8cf60ba..4525bca 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -26,7 +26,7 @@ Undecided, check_repo, ) -from scripts.devstream.stats import ensure_branch, read_json, update_json, write_json +from scripts.devstream.stats import ensure_branch, read_json, update_json from scripts.validate_data import validate CHECKED = {"live", "warned", "archived"} @@ -197,7 +197,7 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: entries = load(path) targets = [e for e in select(entries, repo) if e["status"] in CHECKED] ensure_branch(gh) - rot_state, rot_sha = read_json(gh, "rot.json") + rot_state = read_json(gh, "rot.json")[0] rot_state = rot_state or {} results = {} for e in targets: @@ -215,7 +215,7 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: raise SystemExit(1) save(new, path) elif rot != rot_state: - write_json(gh, "rot.json", rot, rot_sha, "chore(stats): record failing repos") + update_json(gh, "rot.json", lambda _: rot, "chore(stats): record failing repos") # Unchanged entries means the only actions are removal notices, safe to post now rot.json is saved. failed, notified = _post(gh, actions, read_json(gh, "discussions.json")[0] or {}) failed += _mark_notified(gh, notified, rot) @@ -267,6 +267,7 @@ def rekey(data): def record(data): # sync_likes replays these over snapshots taken before the rename; past its 7-day baseline they're unused. kept = {k: v for k, v in data.items() if isinstance(v, dict) and now - _parse(v["at"]) < RENAME_KEEP} + # renames.json keeps one record per old name; a repeat rename replaces it, fine on a weekly schedule. return {**kept, **{old: {"to": new, "at": _iso(now)} for old, new in moves.items()}} numbers = dict(read_json(gh, "discussions.json")[0] or {}) # before the move, so dropped entries find their thread diff --git a/scripts/sync_likes.py b/scripts/sync_likes.py index b56edc6..aae021d 100644 --- a/scripts/sync_likes.py +++ b/scripts/sync_likes.py @@ -22,7 +22,7 @@ fetch_reactors, rank_entries, ) -from scripts.devstream.stats import API, ensure_branch, list_dir, read_json, write_json +from scripts.devstream.stats import API, delete_file, ensure_branch, list_dir, read_json, update_json from scripts.devstream.trust import TrustCache, ensure_block_access # Flagged entries stay listed and ranked; only Trending hides them. @@ -33,6 +33,9 @@ LABELS = ["report", "needs-review"] # How long a reviewed (closed) issue holds off a new one for the same entry. COOLDOWN = {"burst": DAY, "drip": WEEK} +SNAPSHOTS_KEPT = 30 +# A rename's finish normally follows its merge within minutes; past this, sync rather than freeze the board. +RENAME_WAIT = timedelta(hours=3) # timed from likes.json's last commit, which a quiet board may have aged past 3h already def _parse(s: str) -> datetime: @@ -165,6 +168,46 @@ def _report(gh, todo: list[tuple[str, str, str, str]], flags: dict, now: datetim gh.patch(f"{API}/issues/{agg}", _aggregate(listed)) +def _merge_flags(flags: dict, before: dict): + """A mutate for update_json: lay this run's new and changed records over the stored copy. + + Re-read on a 409, so records another writer stored survive, and so do the issue numbers + this run opened: losing one would open the same issue again next hour. + """ + def mutate(cur): + for k, recs in flags.items(): + for kind, r in recs.items(): + if before.get(k, {}).get(kind) != r: + cur.setdefault(k, {})[kind] = r + return cur + return mutate + + +def _rename_in_flight(gh, now, entries, keys, mapping, prev_likes) -> bool: + """True while a rename has merged but rot's finish hasn't moved discussions.json yet. + + The tell: a listed key has no thread, and a key that was ranked last time has a thread but is no + longer listed. An entry removed for good drops out of likes.json, so it can't hold the board. + """ + listed = {normalize_repo(e["repo"]) for e in entries} + moved = [k for k in mapping if k not in listed and k in prev_likes] + if not moved or all(k in mapping for k in keys): + return False + got = gh.get(f"{API}/commits", {"path": "likes.json", "sha": "stats", "per_page": 1}) or [] + at = got[0]["commit"]["committer"]["date"] if got else None + if at is not None and now - _parse(at) <= RENAME_WAIT: + logging.warning("a rename of %s is half done; skipping this sync", ", ".join(moved)) + return True + logging.error("%s still mapped but unlisted after %s; syncing anyway", ", ".join(moved), RENAME_WAIT) + return False + + +def _prune_snapshots(gh, today: str) -> None: + names = sorted({n for n in list_dir(gh, "snapshots") if _snapshot_date(n)} | {today}) + for n in names[:-SNAPSHOTS_KEPT]: + delete_file(gh, f"snapshots/{n}", f"chore(stats): drop snapshot {n.removesuffix('.json')}") + + def _under_review(flags: dict, key: str) -> bool: # Closing the review issue is what clears a flag the burst window has outlived. Drips never flag. rec = flags.get(key, {}).get("burst") @@ -180,16 +223,17 @@ def run(gh, entries, now: datetime, excluded: set[str]) -> None: if e["status"] in RANKED: ranked.setdefault(normalize_repo(e["repo"]), e) keys = sorted(ranked) + prev_likes = read_json(gh, "likes.json")[0] + if _rename_in_flight(gh, now, entries, keys, mapping, prev_likes or {}): + return meta = fetch_meta(gh, keys) reactors = fetch_reactors(gh, sorted({mapping[k] for k in keys if k in mapping})) - prev_likes, likes_sha = read_json(gh, "likes.json") - prev_flags, flags_sha = read_json(gh, "flags.json") + prev_flags = read_json(gh, "flags.json")[0] flags = copy.deepcopy(prev_flags or {}) _review(gh, flags, now) - cache_data, cache_sha = read_json(gh, "trust-cache.json") - cache = _load_cache(cache_data) + cache = _load_cache(read_json(gh, "trust-cache.json")[0]) ensure_block_access(gh) seen = {} @@ -239,9 +283,9 @@ def trusted(login): "flagged": ranked[k]["status"] == "flagged" or k in bursts or _under_review(flags, k), } - prev_meta, meta_sha = read_json(gh, "meta.json") + prev_meta = read_json(gh, "meta.json")[0] snap_path = f"snapshots/{now:%Y-%m-%d}.json" - snap_sha = read_json(gh, snap_path)[1] + flags_write = ("flags.json", _merge_flags(flags, prev_flags or {}), "chore(stats): record Like reviews") if candidates: try: @@ -250,25 +294,22 @@ def trusted(login): # Without the records of issues already opened, the next run would open them again. if flags != (prev_flags or {}): try: - write_json(gh, "flags.json", flags, flags_sha, "chore(stats): record Like reviews") + update_json(gh, *flags_write) except Exception: logging.exception("could not write flags.json") raise - writes = [ - # First, so a later failed write can't lose the record of issues already opened. - ("flags.json", flags, prev_flags or {}, flags_sha, "chore(stats): record Like reviews"), - ("meta.json", meta, prev_meta, meta_sha, "chore(stats): update repo meta"), - ("likes.json", out, prev_likes, likes_sha, "chore(stats): update likes"), - ] - if snap_sha is None: - snap = {k: {"likes": v["likes"], "rank": v["rank"]} for k, v in out.items()} - writes.append((snap_path, snap, None, None, f"chore(stats): snapshot likes for {now:%Y-%m-%d}")) - writes.append(("trust-cache.json", cache.entries, cache_data, cache_sha, "chore(stats): update trust cache")) - for path, data, before, sha, message in writes: - if data != before: - write_json(gh, path, data, sha, message) - + snap = {k: {"likes": v["likes"], "rank": v["rank"]} for k, v in out.items()} + # Each is re-applied onto a fresh read if another writer 409s it first. + update_json(gh, *flags_write) # first, so a later failed write can't lose the record of issues opened + # Built from the file as it was read; if someone else changed it since, theirs stands until next hour. + update_json(gh, "meta.json", lambda cur: meta if cur == (prev_meta or {}) else cur, "chore(stats): update repo meta") + update_json(gh, "likes.json", lambda cur: out if cur == (prev_likes or {}) else cur, "chore(stats): update likes") + update_json(gh, snap_path, lambda cur: cur or snap, f"chore(stats): snapshot likes for {now:%Y-%m-%d}") + # Past the TTL a record is never read again, and unpruned the file passes 1 MB. + fresh = {k: v for k, v in cache.entries.items() if cache.get(k, now) is not None} + update_json(gh, "trust-cache.json", lambda _: fresh, "chore(stats): update trust cache", compact=True) + _prune_snapshots(gh, snap_path.removeprefix("snapshots/")) def main(): excluded = {u.strip().lower() for u in os.environ.get("EXCLUDED_USERS", "").split(",") if u.strip()} diff --git a/scripts/tests/fake_gh.py b/scripts/tests/fake_gh.py index a05ccac..be6f2a1 100644 --- a/scripts/tests/fake_gh.py +++ b/scripts/tests/fake_gh.py @@ -118,8 +118,8 @@ def patch(self, path, body): def put(self, path, body): return self._write("PUT", path, body) - def delete(self, path): - return self._write("DELETE", path, None) + def delete(self, path, body=None): + return self._write("DELETE", path, body) def graphql(self, query: str, variables: dict, allow_not_found: bool = False) -> dict: self.last_redirect = None diff --git a/scripts/tests/test_gh.py b/scripts/tests/test_gh.py index cf7a2d9..07f11b8 100644 --- a/scripts/tests/test_gh.py +++ b/scripts/tests/test_gh.py @@ -1,5 +1,7 @@ import json +import socket import threading +import urllib.error import time import unittest from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer @@ -266,6 +268,121 @@ def test_second_rate_limit_raises(self): self.assertEqual(len(self.slept), 1) +class Retries(unittest.TestCase): + """_send is scripted: each item is a status code or an exception to raise.""" + + def gh(self, *script): + slept = [] + gh = GitHub("tok", "http://x", sleep=slept.append) + sent = [] + + def send(method, url, body): + sent.append(method) + r = script[len(sent) - 1] + if isinstance(r, BaseException): + raise r + return r, {}, b'{"ok": true}' if r < 300 else b"{}" + gh._send = send + return gh, sent, slept + + def test_get_502_then_200_succeeds(self): + gh, sent, slept = self.gh(502, 200) + self.assertEqual(gh.get("/a"), {"ok": True}) + self.assertEqual((len(sent), len(slept)), (2, 1)) + + def test_three_503s_raise(self): + gh, sent, slept = self.gh(503, 503, 503, 200) + with self.assertRaises(GitHubError) as c: + gh.get("/a") + self.assertEqual((c.exception.status, len(sent), len(slept)), (503, 3, 2)) + self.assertLess(slept[0], slept[1]) + + def test_404_not_retried(self): + gh, sent, _ = self.gh(404, 200) + self.assertIsNone(gh.get("/a")) + self.assertEqual(len(sent), 1) + + def test_422_not_retried(self): + gh, sent, _ = self.gh(422, 200) + with self.assertRaises(GitHubError): + gh.post("/a", {}) + self.assertEqual(len(sent), 1) + + def test_post_500_not_retried(self): + gh, sent, _ = self.gh(500, 200) + with self.assertRaises(GitHubError): + gh.post("/a", {}) + self.assertEqual(len(sent), 1) + + def test_post_502_not_retried(self): + # The issue or comment may already exist; a second POST would duplicate it. + gh, sent, _ = self.gh(502, 200) + with self.assertRaises(GitHubError): + gh.post("/a", {}) + self.assertEqual(len(sent), 1) + + def test_put_patch_delete_502_retried(self): + for m, args in (("put", ({},)), ("patch", ({},)), ("delete", ())): + gh, sent, _ = self.gh(502, 200) + self.assertEqual(getattr(gh, m)("/a", *args), {"ok": True}) + self.assertEqual(len(sent), 2, m) + + def test_graphql_query_502_retried(self): + gh, sent, _ = self.gh(502, 200) + gh.graphql("query Q { x }", {}) + self.assertEqual(len(sent), 2) + + def test_graphql_query_timeout_retried(self): + gh, sent, _ = self.gh(TimeoutError(), 200) + gh.graphql(" query Q { x }", {}) + self.assertEqual(len(sent), 2) + + def test_graphql_mutation_502_not_retried(self): + for err in (502, TimeoutError()): + gh, sent, _ = self.gh(err, 200) + with self.assertRaises((GitHubError, TimeoutError)): + gh.graphql("mutation M { x }", {}) + self.assertEqual(len(sent), 1) + + def test_graphql_mutation_retried_when_never_sent(self): + gh, sent, _ = self.gh(urllib.error.URLError(ConnectionRefusedError()), 200) + gh.graphql("\n mutation M { x }", {}) + self.assertEqual(len(sent), 2) + + def test_get_network_error_retried(self): + for err in (urllib.error.URLError("down"), TimeoutError(), ConnectionResetError()): + gh, sent, _ = self.gh(err, 200) + self.assertEqual(gh.get("/a"), {"ok": True}) + self.assertEqual(len(sent), 2) + + def test_three_network_errors_raise(self): + gh, sent, _ = self.gh(TimeoutError(), TimeoutError(), TimeoutError(), 200) + with self.assertRaises(TimeoutError): + gh.get("/a") + self.assertEqual(len(sent), 3) + + def test_write_retried_only_when_never_sent(self): + refused = urllib.error.URLError(ConnectionRefusedError()) + dns = urllib.error.URLError(socket.gaierror()) + for err in (refused, dns): + gh, sent, _ = self.gh(err, 200) + self.assertEqual(gh.post("/a", {}), {"ok": True}) + self.assertEqual(len(sent), 2) + for err in (TimeoutError(), ConnectionResetError(), urllib.error.URLError("reset")): + gh, sent, _ = self.gh(err, 200) + with self.assertRaises(OSError): + gh.patch("/a", {}) + self.assertEqual(len(sent), 1) + + def test_rate_limit_rules_unchanged(self): + gh, sent, _ = self.gh(200) + gh._send = lambda m, u, b: (sent.append(m), (403, {"x-ratelimit-remaining": "0", + "x-ratelimit-reset": str(LONG)}, b"{}"))[1] + with self.assertRaises(RateLimited): + gh.get("/a") + self.assertEqual(len(sent), 1) + + class FakeTest(unittest.TestCase): def test_same_public_methods(self): pub = lambda c: {n for n in dir(c) if not n.startswith("_") and callable(getattr(c, n))} diff --git a/scripts/tests/test_likes.py b/scripts/tests/test_likes.py index 7d006b2..65ffd09 100644 --- a/scripts/tests/test_likes.py +++ b/scripts/tests/test_likes.py @@ -65,7 +65,8 @@ def __init__(self, entries, discussions, users=None, stats=None, stars=None, iss "content": base64.b64encode(json.dumps(data).encode()).decode(), "sha": f"sha-{path}"} snaps = sorted(p for p in self.stats if p.startswith("snapshots/")) if snaps: - routes[f"GET {API}/contents/snapshots"] = [{"name": p.split("/")[1], "type": "file"} for p in snaps] + routes[f"GET {API}/contents/snapshots"] = [ + {"name": p.split("/")[1], "type": "file", "sha": f"sha-{p}"} for p in snaps] days = [f"snapshots/{NOW + timedelta(days=d):%Y-%m-%d}.json" for d in range(12)] for path in ("meta.json", "likes.json", "flags.json", "trust-cache.json", *days): routes[f"PUT {API}/contents/{path}"] = {} @@ -755,5 +756,128 @@ def test_missing_stats_branch(self): self.assertEqual(w.puts(), []) +class Conflicts(unittest.TestCase): + """A 409 on the stats branch means another writer got there first: re-read, re-apply, write again.""" + + def conflict_once(self, w, path, then=None): + n = [] + + def put(_): + n.append(1) + if len(n) == 1: + if then: + then() + raise GitHubError(409, "conflict") + return {} + w.routes[f"PUT {API}/contents/{path}"] = put + return n + + def test_flags_409_then_success_keeps_both_records(self): + w = World([entry("o/a")], {"o/a": burst_nodes(10, 10)}) + other = {"o/z": {"burst": {"kind": "burst", "issue": 5, "opened_at": iso(NOW), "reviewed_at": None}}} + + def someone_wrote(): + w.routes[f"GET {API}/contents/flags.json"] = { + "content": base64.b64encode(json.dumps(other).encode()).decode(), "sha": "sha-new"} + n = self.conflict_once(w, "flags.json", someone_wrote) + w.run() + self.assertEqual(len(n), 2) + self.assertEqual(titles(w), ["Like burst: o/a"]) + got = w.written("flags.json") + self.assertEqual((got["o/a"]["burst"]["issue"], got["o/z"]["burst"]["issue"]), (100, 5)) + self.assertEqual(w.puts()[-5][2]["sha"], "sha-new") + + def test_flags_409_then_success_opens_no_duplicate_next_run(self): + w = World([entry("o/a")], {"o/a": burst_nodes(10, 10)}) + self.conflict_once(w, "flags.json") + w.run() + w.persist() + w.run(now=NOW + timedelta(minutes=5)) + self.assertEqual(w.issues_opened(), []) + + def test_likes_and_meta_409_after_someone_else_wrote_skips_this_hour(self): + w = World([entry("o/a")], {"o/a": [node("x"), node("y")]}) + theirs = {"o/a": {"likes": 9}} + for path in ("likes.json", "meta.json"): + def someone_wrote(path=path): + w.routes[f"GET {API}/contents/{path}"] = { + "content": base64.b64encode(json.dumps(theirs).encode()).decode(), "sha": "sha-theirs"} + n = self.conflict_once(w, path, someone_wrote) + w.run() + self.assertEqual(len(n), 1, path) + w.gh.calls.clear() + + def test_likes_409_then_success(self): + w = World([entry("o/a")], {"o/a": [node("x"), node("y")]}) + n = self.conflict_once(w, "likes.json") + w.run() + self.assertEqual(len(n), 2) + self.assertEqual(w.written("likes.json")["o/a"]["likes"], 2) + + +class Size(unittest.TestCase): + def test_trust_cache_pruned_past_ttl_and_compact(self): + stale = {"checked_at": iso(NOW - timedelta(days=8)), "trusted": True, "reason": None, "created_at": OLD} + w = World([entry("o/a")], {"o/a": [node("x")]}, stats={"trust-cache.json": {"gone": stale}}) + w.run() + self.assertEqual(list(w.written("trust-cache.json")), ["x"]) + put = [b for _, p, b in w.puts() if p.endswith("trust-cache.json")][0] + self.assertNotIn("\n ", base64.b64decode(put["content"]).decode()) + + def test_only_the_last_30_snapshots_are_kept(self): + stats = {f"snapshots/{NOW - timedelta(days=d):%Y-%m-%d}.json": {} for d in range(1, 33)} + w = World([entry("o/a")], {"o/a": [node("x")]}, stats=stats) + for p in stats: + w.routes[f"DELETE {API}/contents/{p}"] = None + w.run() + deletes = [(i, c) for i, c in enumerate(w.gh.calls) if c[0] == "DELETE"] + old = [f"{API}/contents/snapshots/{NOW - timedelta(days=d):%Y-%m-%d}.json" for d in (32, 31, 30)] + self.assertEqual([c[1] for _, c in deletes], old) + self.assertEqual(deletes[0][1][2]["sha"], f"sha-snapshots/{NOW - timedelta(days=32):%Y-%m-%d}.json") + snap_put = max(i for i, c in enumerate(w.gh.calls) if c[0] == "PUT" and c[1].endswith(f"{TODAY}.json")) + self.assertGreater(deletes[0][0], snap_put) + + def test_30_or_fewer_snapshots_deletes_nothing(self): + stats = {f"snapshots/{NOW - timedelta(days=d):%Y-%m-%d}.json": {} for d in range(1, 30)} + w = World([entry("o/a")], {"o/a": [node("x")]}, stats=stats) + w.run() + self.assertFalse([c for c in w.gh.calls if c[0] == "DELETE"]) + + +class HalfDoneRename(unittest.TestCase): + ROW = {"likes": 2, "delta_7d": 0, "rank": 1, "rank_delta_7d": 0, "flagged": False} + + def world(self, likes_age): + # The rename PR merged but rot's finish hasn't moved discussions.json yet. + w = World([entry("o/new")], {"o/old": [node("x"), node("y")]}, stats={"likes.json": {"o/old": self.ROW}}) + w.routes[f"GET {API}/commits"] = [{"commit": {"committer": {"date": iso(NOW - likes_age)}}}] + return w + + def writes(self, w): + return [c for c in w.gh.calls if c[0] in ("PUT", "POST", "PATCH", "DELETE")] + + def test_rename_within_3_hours_writes_nothing(self): + w = self.world(timedelta(hours=2)) + w.run() + self.assertEqual(self.writes(w), []) + + def test_rename_after_3_hours_syncs_anyway(self): + w = self.world(timedelta(hours=4)) + with self.assertLogs(level="ERROR"): + w.run() + self.assertIn("o/new", w.written("likes.json")) + + def test_removed_entry_and_new_entry_still_sync(self): + # o/gone was removed long ago: it isn't in likes.json, so it's no rename in flight. + w = World([entry("o/a"), entry("o/b")], {"o/a": [node("x")], "o/gone": [node("y")]}, + stats={"likes.json": {"o/a": self.ROW}}) + w.run() + self.assertEqual(sorted(w.written("likes.json")), ["o/a", "o/b"]) + + def test_new_entry_without_thread_still_syncs(self): + w = World([entry("o/a"), entry("o/b")], {"o/a": [node("x")]}) + w.run() + self.assertEqual(w.written("likes.json")["o/b"]["likes"], 0) + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_rot.py b/scripts/tests/test_rot.py index 88f2c7f..da4675f 100644 --- a/scripts/tests/test_rot.py +++ b/scripts/tests/test_rot.py @@ -522,7 +522,7 @@ def test_notice_in_finish_marks_the_record_notified(self): {"alice/tool": rec})) self.assertEqual(state["rot.json"], {"alice/tool": {**rec, "notified": True}}) - def test_stats_write_retries_once_after_409(self): + def test_stats_write_retries_after_409(self): r, state = store({"discussions.json": {"alice/tool": 5}, "flags.json": {"alice/tool": {"burst": 1}}}, threads({5: "alice/tool"}), conflicts={"flags.json": 1}) finish(FakeGitHub(r), self.plan([Action("rename", "alice/tool-ng", "", "alice/tool")])) @@ -533,7 +533,7 @@ def test_stats_write_retries_once_after_409(self): def test_comments_still_post_when_a_stats_write_fails(self): titles = {5: "alice/tool"} r, state = store({"discussions.json": {"alice/tool": 5, "bob/other": 6}, "flags.json": {"alice/tool": {}}}, - threads(titles), conflicts={"flags.json": 2}) + threads(titles), conflicts={"flags.json": 3}) g = FakeGitHub(r) with self.assertRaises(SystemExit): finish(g, self.plan([Action("rename", "alice/tool-ng", "", "alice/tool"), @@ -547,7 +547,7 @@ def test_failed_rot_write_never_starts_the_clock_on_an_old_record(self): _, actions, rot = plan_rot([entry()], {"alice/tool": gone()}, NOW, {"alice/tool": old_rec}) self.assertEqual(kinds(actions), [("notice", "alice/tool")]) r, state = store({"rot.json": {"alice/tool": old_rec}, "discussions.json": {"alice/tool": 5}}, - threads({5: "alice/tool"}), conflicts={"rot.json": 2}) + threads({5: "alice/tool"}), conflicts={"rot.json": 3}) g = FakeGitHub(r) with self.assertRaises(SystemExit): finish(g, {"changed": True, "actions": [a.__dict__ for a in actions], "rot_state": rot}) @@ -560,7 +560,7 @@ def test_failed_rot_write_never_starts_the_clock_on_an_old_record(self): def test_no_retitle_when_the_key_move_fails(self): titles = {5: "alice/tool", 6: "bob/kept"} r, state = store({"discussions.json": {"alice/tool": 5, "bob/kept": 6}}, threads(titles), - conflicts={"discussions.json": 2}) + conflicts={"discussions.json": 3}) g = FakeGitHub(r) with self.assertRaises(SystemExit): finish(g, self.plan([Action("rename", "alice/tool-ng", "", "alice/tool"), diff --git a/scripts/tests/test_stats.py b/scripts/tests/test_stats.py index 838eaa4..375778e 100644 --- a/scripts/tests/test_stats.py +++ b/scripts/tests/test_stats.py @@ -1,7 +1,9 @@ +import base64 +import json import unittest from scripts.devstream.gh import GitHubError -from scripts.devstream.stats import API, list_dir +from scripts.devstream.stats import API, delete_file, list_dir, read_json, update_json from scripts.tests.fake_gh import FakeGitHub @@ -19,3 +21,80 @@ def test_non_list_raises(self): gh = FakeGitHub({f"GET {API}/contents/d": {"name": "d", "type": "file", "content": ""}}) with self.assertRaises(GitHubError): list_dir(gh, "d") + + +class ReadJson(unittest.TestCase): + def test_file_over_1mb_is_read_through_the_blob(self): + raw = base64.encodebytes(json.dumps({"a": 1}).encode()).decode() # the blobs API wraps lines + gh = FakeGitHub({ + f"GET {API}/contents/big.json": {"content": "", "encoding": "none", "sha": "s1"}, + f"GET {API}/git/blobs/s1": {"content": raw, "encoding": "base64", "sha": "s1"}, + }) + self.assertEqual(read_json(gh, "big.json"), ({"a": 1}, "s1")) + + def test_missing_blob_raises(self): + gh = FakeGitHub({f"GET {API}/contents/big.json": {"content": "", "encoding": "none", "sha": "s1"}}) + with self.assertRaises(GitHubError): + read_json(gh, "big.json") + + +class Compact(unittest.TestCase): + def test_compact_write_has_no_indent(self): + gh = FakeGitHub({f"PUT {API}/contents/c.json": {}}) + update_json(gh, "c.json", lambda _: {"a": {"b": 1}}, "m", compact=True) + body = base64.b64decode(gh.calls[-1][2]["content"]).decode() + self.assertEqual(body, '{"a":{"b":1}}\n') + + +class DeleteFile(unittest.TestCase): + def test_deletes_with_sha_on_stats_branch(self): + gh = FakeGitHub({f"GET {API}/contents/s/a.json": {"content": "", "sha": "s9"}, + f"DELETE {API}/contents/s/a.json": None}) + delete_file(gh, "s/a.json", "chore: drop") + body = gh.calls[-1][2] + self.assertEqual((body["sha"], body["branch"], body["message"]), ("s9", "stats", "chore: drop")) + self.assertIn("committer", body) + + def test_retries_once_on_409(self): + n = [] + + def conflict(_): + n.append(1) + if len(n) == 1: + raise GitHubError(409, "conflict") + gh = FakeGitHub({f"GET {API}/contents/a.json": {"content": "", "sha": "s9"}, + f"DELETE {API}/contents/a.json": conflict}) + delete_file(gh, "a.json", "m") + self.assertEqual(len(n), 2) + + def test_already_gone_is_fine(self): + delete_file(FakeGitHub({}), "a.json", "m") + + +class UpdateJson(unittest.TestCase): + def puts(self, fails): + n = [] + + def put(_): + n.append(1) + if len(n) <= fails: + raise GitHubError(409, "conflict") + return {} + return FakeGitHub({f"PUT {API}/contents/a.json": put}), n + + def test_two_409s_then_success(self): + gh, n = self.puts(2) + update_json(gh, "a.json", lambda _: {"a": 1}, "m") + self.assertEqual(len(n), 3) + + def test_three_409s_raise(self): + gh, n = self.puts(3) + with self.assertRaises(GitHubError): + update_json(gh, "a.json", lambda _: {"a": 1}, "m") + self.assertEqual(len(n), 3) + + +class DeleteGone(unittest.TestCase): + def test_404_on_delete_is_already_gone(self): + gh = FakeGitHub({f"GET {API}/contents/a.json": {"content": "", "sha": "s9"}}) # no DELETE route: 404 + delete_file(gh, "a.json", "m") diff --git a/site/tests/data.test.mjs b/site/tests/data.test.mjs index e75306b..6b1ab8b 100644 --- a/site/tests/data.test.mjs +++ b/site/tests/data.test.mjs @@ -53,6 +53,7 @@ test("sort by likes then stars then name", () => { "quiet-co/tiny-queue", "Pixel-Forge/shader-kit", "burst/likes-spike", + "future/fresh-entry", "evil/hostile-stats", "fresh/no-stats-yet", ]); @@ -78,7 +79,10 @@ test("archived and superseded appear only in the archive list", () => { }); test("new sorts by submission time, newest first", () => { - assert.equal(newest(all().entries)[0].repo, "fresh/no-stats-yet"); + assert.deepEqual(names(newest(all().entries)).slice(0, 2), [ + "future/fresh-entry", + "fresh/no-stats-yet", + ]); }); test("stats are merged by lowercased repo key", () => { @@ -110,7 +114,7 @@ test("missing stats files are treated as empty", () => { dataDir: fixtures, statsDir: `${fixtures}/does-not-exist`, }); - assert.equal(liveList(entries).length, 7); + assert.equal(liveList(entries).length, 8); assert.ok(entries.every((e) => e.likes === 0 && e.rank === null)); }); diff --git a/site/tests/dist.test.mjs b/site/tests/dist.test.mjs index e42fb9b..4edf847 100644 --- a/site/tests/dist.test.mjs +++ b/site/tests/dist.test.mjs @@ -105,7 +105,7 @@ test("top10.svg and badge.json are built", () => { assert.ok( svg.startsWith(" { @@ -139,6 +139,14 @@ test("the Trending tab marks exactly the rows trending() allows", async () => { assert.ok(rows.length > 0, page); // zz-tie/same-likes climbs by 3, under TRENDING_MIN_DELTA, so the hero would skip it. assert.ok(!marked.includes("zz-tie/same-likes"), page); + // future/fresh-entry climbs by 9 but isn't 7 days old and has no seed flag: the age gate. + if (page === "index.html") { + assert.ok( + rows.some((r) => r.includes('data-q="future/fresh-entry ')), + page, + ); + } + assert.ok(!marked.includes("future/fresh-entry"), page); assert.deepEqual( marked.sort(), want.filter((r) => rows.some((x) => x.includes(`data-q="${r} `))).sort(), diff --git a/site/tests/embeds.test.mjs b/site/tests/embeds.test.mjs index d2be268..f0d3662 100644 --- a/site/tests/embeds.test.mjs +++ b/site/tests/embeds.test.mjs @@ -54,7 +54,7 @@ test("badge.json has the shields.io endpoint shape", () => { assert.deepEqual(badge(live), { schemaVersion: 1, label: "listed", - message: "7 repos", + message: "8 repos", color: BRAND_HEX, }); assert.equal(badge(live.slice(0, 1)).message, "1 repo"); diff --git a/site/tests/fixtures/entries.json b/site/tests/fixtures/entries.json index f92521f..7159d42 100644 --- a/site/tests/fixtures/entries.json +++ b/site/tests/fixtures/entries.json @@ -120,6 +120,18 @@ "warned_at": null, "superseded_by": null, "added_by_seed": false + }, + { + "repo": "future/fresh-entry", + "category": "testing", + "description": "Climbing fast but too new for Trending.", + "submitted_by": "fay", + "submitted_at": "2999-01-01T00:00:00Z", + "issue": 11, + "status": "live", + "warned_at": null, + "superseded_by": null, + "added_by_seed": false } ] } diff --git a/site/tests/fixtures/stats/likes.json b/site/tests/fixtures/stats/likes.json index 4ff4725..259eab5 100644 --- a/site/tests/fixtures/stats/likes.json +++ b/site/tests/fixtures/stats/likes.json @@ -47,5 +47,12 @@ "rank": "