diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 18e396f..b1f08db 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,13 +1,15 @@ name: Deploy site # Builds the leaderboard from main plus the stats branch and ships it to -# Cloudflare Pages, after every merge, Likes sync and site change. +# Cloudflare Pages, after every merge, Likes sync and site change, and again once +# a submission or health check has made its threads and moved its stats. on: push: branches: [main] paths: ["site/**", "data/**"] workflow_run: - workflows: ["Create entry Discussions", "Sync Likes"] + workflows: + ["Create entry Discussions", "Sync Likes", "Submit", "Health check"] types: [completed] workflow_dispatch: diff --git a/.github/workflows/on-merge.yml b/.github/workflows/on-merge.yml index 6a0c41d..9473c08 100644 --- a/.github/workflows/on-merge.yml +++ b/.github/workflows/on-merge.yml @@ -1,11 +1,10 @@ name: Create entry Discussions -# Every listed entry gets a Discussion; a thumbs-up on it is a Like. The -# hourly run picks up whatever the 40-per-run cap left behind. +# Every listed entry gets a Discussion; a thumbs-up on it is a Like. Submit and +# the health check make threads themselves, right after their PR merges; this +# hourly run is the backstop for the 40-per-run cap, failed creates and any +# other change to the list. on: - push: - branches: [main] - paths: [data/entries.json] schedule: - cron: "23 * * * *" workflow_dispatch: @@ -14,7 +13,7 @@ permissions: {} # Shared with rot.yml so this waits until a rename's keys have moved and its # thread is retitled; otherwise it would open a second thread for the new name. -# A run dropped from the queue is picked up by the hourly schedule. +# A run dropped from the queue costs nothing: the next one covers the same list. concurrency: group: list-writes cancel-in-progress: false diff --git a/.github/workflows/rot.yml b/.github/workflows/rot.yml index 6e39fe5..de4c782 100644 --- a/.github/workflows/rot.yml +++ b/.github/workflows/rot.yml @@ -92,3 +92,8 @@ jobs: env: GITHUB_TOKEN: ${{ steps.token.outputs.token }} run: python -m scripts.rot finish --plan "$RUNNER_TEMP/plan.json" + # Only after finish has moved a renamed entry's thread to its new key, or this would open a second one. + - if: steps.plan.outputs.changed == 'true' + env: + GITHUB_TOKEN: ${{ steps.token.outputs.token }} + run: python -m scripts.on_merge diff --git a/.github/workflows/submit.yml b/.github/workflows/submit.yml index 2cb223b..bc676ad 100644 --- a/.github/workflows/submit.yml +++ b/.github/workflows/submit.yml @@ -171,6 +171,22 @@ jobs: python -m scripts.submit comment --result "$RESULT" --kind accepted > "$RUNNER_TEMP/comment.md" gh issue comment "$ISSUE" --body-file "$RUNNER_TEMP/comment.md" gh issue edit "$ISSUE" --add-label accepted + # The new entry's Like thread, made while this run still holds list-writes so a burst of + # submissions can't starve it. A failure is left to the hourly on-merge run, not parked. + - id: discussions_token + if: steps.apply.outputs.branch != '' + continue-on-error: true + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + permission-contents: write + permission-discussions: write + - if: steps.apply.outputs.branch != '' && steps.discussions_token.outputs.token != '' + continue-on-error: true + env: + GITHUB_TOKEN: ${{ steps.discussions_token.outputs.token }} + run: python -m scripts.on_merge - if: steps.apply.outputs.branch == '' env: GH_TOKEN: ${{ steps.issues_token.outputs.token }} diff --git a/scripts/rot.py b/scripts/rot.py index 5c9f3ee..8cf60ba 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -35,6 +35,7 @@ CONFIRM = timedelta(days=6) # An older sighting is a different episode, so it starts the week of grace again. STALE = timedelta(days=14) +RENAME_KEEP = timedelta(days=8) REMOVAL_REASONS = (MSG_GONE, MSG_LICENSE, MSG_NO_README, MSG_SHORT_README) REPO_INPUT = re.compile(r"[A-Za-z0-9-]{1,39}/[A-Za-z0-9._-]{1,100}") @@ -245,7 +246,7 @@ def _move(data: dict, moves: dict[str, str]) -> dict: return out -def finish(gh, plan: dict) -> None: +def finish(gh, plan: dict, now: datetime | None = None) -> None: """Write stats, then comment and retitle. Run only once the entries PR has merged. A failed stats write doesn't hold back the comments; the run fails at the end instead. @@ -261,6 +262,13 @@ def rekey(data): data.pop(k, None) return _move(data, moves) + now = now or datetime.now(timezone.utc) + + def record(data): + # sync_likes replays these over snapshots taken before the rename; past its 7-day baseline they're unused. + kept = {k: v for k, v in data.items() if isinstance(v, dict) and now - _parse(v["at"]) < RENAME_KEEP} + return {**kept, **{old: {"to": new, "at": _iso(now)} for old, new in moves.items()}} + numbers = dict(read_json(gh, "discussions.json")[0] or {}) # before the move, so dropped entries find their thread failed = 0 moved = True @@ -268,6 +276,10 @@ def rekey(data): ("rot.json", lambda _: plan["rot_state"], "chore(stats): record failing repos"), ("discussions.json", rekey, "chore(stats): follow renamed repos"), ("flags.json", rekey, "chore(stats): follow renamed repos"), + # The site build reads these two by key, so they move now rather than at the next hourly sync. + ("likes.json", rekey, "chore(stats): follow renamed repos"), + ("meta.json", rekey, "chore(stats): follow renamed repos"), + ("renames.json", record, "chore(stats): record renamed repos"), ): try: update_json(gh, path, mutate, message) diff --git a/scripts/sync_likes.py b/scripts/sync_likes.py index a8d110e..b56edc6 100644 --- a/scripts/sync_likes.py +++ b/scripts/sync_likes.py @@ -120,8 +120,11 @@ def _snapshot_date(name: str): return None -def _baseline(gh, now: datetime) -> dict | None: - """The snapshot from 7 days ago, else the oldest one between 1 and 7 days old, else None.""" +def _baseline(gh, now: datetime, renames: dict) -> dict | None: + """The snapshot from 7 days ago, else the oldest one between 1 and 7 days old, else None. + + Keys are moved through renames.json, so a renamed entry is compared with its own past. + """ week, day = (now - WEEK).date(), (now - DAY).date() dates = sorted(d for n in list_dir(gh, "snapshots") if (d := _snapshot_date(n))) old = [d for d in dates if week < d <= day] @@ -132,6 +135,10 @@ def _baseline(gh, now: datetime) -> dict | None: if data is None: # Listed but unreadable: fail closed rather than read it as an empty baseline. raise GitHubError(404, f"snapshot {pick} is listed but could not be read") + # Oldest first, so a->b then b->c lands on c. A rename before the snapshot is already in it. + for old, r in sorted(renames.items(), key=lambda kv: kv[1]["at"]): + if _parse(r["at"]).date() >= pick and old in data: + data[r["to"]] = data.pop(old) return data @@ -219,7 +226,7 @@ def trusted(login): candidates.sort(key=lambda c: c[0] != "burst") ranks = rank_entries(likes, {k: m["stars"] for k, m in meta.items()}) - old = _baseline(gh, now) + old = _baseline(gh, now, read_json(gh, "renames.json")[0] or {}) out = {} for k in keys: o = (old or {}).get(k) diff --git a/scripts/tests/test_likes.py b/scripts/tests/test_likes.py index a7769b7..7d006b2 100644 --- a/scripts/tests/test_likes.py +++ b/scripts/tests/test_likes.py @@ -649,6 +649,52 @@ def test_snapshot_written_once_per_day(self): self.assertEqual(w2.written("likes.json")["o/a"]["likes"], 2) +class Renames(unittest.TestCase): + def snap(self, data): + return {f"snapshots/{WEEK_AGO}.json": data} + + def test_baseline_follows_a_rename(self): + w = World([entry("o/new")], {"o/new": [node("x"), node("y"), node("z"), node("w")]}, + stats={**self.snap({"o/old": {"likes": 3, "rank": 2}}), + "renames.json": {"o/old": {"to": "o/new", "at": iso(NOW - timedelta(days=2))}}}) + w.run() + got = w.written("likes.json")["o/new"] + self.assertEqual((got["delta_7d"], got["rank_delta_7d"]), (1, 1)) + + def test_chained_renames_replay_in_order(self): + w = World([entry("o/c")], {"o/c": [node("x"), node("y")]}, + stats={**self.snap({"o/a": {"likes": 1, "rank": 1}}), + "renames.json": {"o/b": {"to": "o/c", "at": iso(NOW - timedelta(days=1))}, + "o/a": {"to": "o/b", "at": iso(NOW - timedelta(days=3))}}}) + w.run() + self.assertEqual(w.written("likes.json")["o/c"]["delta_7d"], 1) + + def test_rename_older_than_the_baseline_is_not_replayed(self): + # The snapshot already holds the new name, and the old one now belongs to someone else. + w = World([entry("o/a"), entry("o/new")], {"o/a": [node("x")], "o/new": [node("x"), node("y")]}, + stats={**self.snap({"o/a": {"likes": 0, "rank": 2}, "o/new": {"likes": 2, "rank": 1}}), + "renames.json": {"o/a": {"to": "o/new", "at": iso(NOW - timedelta(days=9))}}}) + w.run() + likes = w.written("likes.json") + self.assertEqual((likes["o/a"]["delta_7d"], likes["o/new"]["delta_7d"]), (1, 0)) + + def test_rename_then_sync_keeps_likes_and_trend(self): + from scripts.rot import Action, finish + from scripts.tests.test_rot import store, threads + + row = {"likes": 3, "delta_7d": 0, "rank": 1, "rank_delta_7d": 0, "flagged": False} + r, state = store({"discussions.json": {"o/old": 1}, "likes.json": {"o/old": row}}, threads({1: "o/old"})) + plan = {"changed": True, "actions": [Action("rename", "o/new", "", "o/old").__dict__], "rot_state": {}} + finish(FakeGitHub(r), plan, now=NOW - timedelta(days=2)) + self.assertEqual(state["likes.json"], {"o/new": row}) + + w = World([entry("o/new")], {"o/new": [node("x"), node("y"), node("z"), node("w")]}, + stats={**state, **self.snap({"o/old": {"likes": 3, "rank": 1}})}) + w.run() + got = w.written("likes.json")["o/new"] + self.assertEqual((got["likes"], got["delta_7d"], got["rank_delta_7d"]), (4, 1, 0)) + + class Writes(unittest.TestCase): def test_write_order_and_commit_shape(self): w = World([entry("o/a")], {"o/a": [node("x")]}) diff --git a/scripts/tests/test_on_merge.py b/scripts/tests/test_on_merge.py index 6cc2683..56d4ad8 100644 --- a/scripts/tests/test_on_merge.py +++ b/scripts/tests/test_on_merge.py @@ -1,6 +1,7 @@ import base64 import json import unittest +from pathlib import Path from scripts.devstream.gh import GitHubError from scripts.on_merge import CAP, run @@ -149,5 +150,43 @@ def test_no_put_when_unchanged(self): self.assertEqual(puts(g), []) +WORKFLOWS = Path(__file__).resolve().parents[2] / ".github" / "workflows" + + +def step_after(text, marker): + """The text from marker to the end of the file.""" + return text[text.index(marker):] + + +class Workflows(unittest.TestCase): + """Threads are made by the run that changed the list, while it still holds list-writes.""" + + def test_rot_makes_threads_after_its_keys_move(self): + text = (WORKFLOWS / "rot.yml").read_text() + self.assertIn("python -m scripts.on_merge", step_after(text, "python -m scripts.rot finish")) + + def test_submit_makes_the_thread_once_its_pr_has_merged(self): + text = (WORKFLOWS / "submit.yml").read_text() + apply = text[text.index("\n apply:\n"):text.index("\n reject:\n")] + tail = step_after(apply, "did not merge within 15 minutes") + self.assertIn("python -m scripts.on_merge", tail) + # A failed create is the hourly run's to retry; it mustn't park an accepted issue as needs-review. + step = tail[tail.rindex("- ", 0, tail.index("python -m scripts.on_merge")):] + self.assertIn("continue-on-error: true", step[:step.index("python -m scripts.on_merge")]) + + def test_on_merge_is_only_the_hourly_backstop(self): + text = (WORKFLOWS / "on-merge.yml").read_text() + on = text[text.index("\non:\n"):text.index("\npermissions:")] + self.assertNotIn("push:", on) + self.assertIn("schedule:", on) + self.assertIn("group: list-writes", text) + + def test_site_redeploys_after_the_runs_that_make_threads(self): + text = (WORKFLOWS / "deploy.yml").read_text() + after = text[text.index("workflow_run:"):text.index("types: [completed]")] + for name in ("Submit", "Health check", "Create entry Discussions", "Sync Likes"): + self.assertIn(f'"{name}"', after) + + if __name__ == "__main__": unittest.main() diff --git a/scripts/tests/test_rot.py b/scripts/tests/test_rot.py index f0c781b..88f2c7f 100644 --- a/scripts/tests/test_rot.py +++ b/scripts/tests/test_rot.py @@ -209,6 +209,7 @@ def file(data): f"PUT {API}/contents/rot.json": None, f"PUT {API}/contents/discussions.json": None, f"PUT {API}/contents/flags.json": None, + f"PUT {API}/contents/renames.json": None, f"GET {API}/git/ref/heads/stats": {"ref": "refs/heads/stats"}, } r.update(extra or {}) @@ -241,7 +242,7 @@ def answer(body): return answer r = {f"GET {API}/git/ref/heads/stats": {"ref": "refs/heads/stats"}} - for path in ("rot.json", "discussions.json", "flags.json"): + for path in ("rot.json", "discussions.json", "flags.json", "likes.json", "meta.json", "renames.json"): r[f"GET {API}/contents/{path}"] = get(path) r[f"PUT {API}/contents/{path}"] = put(path) r.update(extra or {}) @@ -483,6 +484,20 @@ def test_rename_moves_stats_keys(self): first_gql = min(i for i, c in enumerate(g.calls) if c[0] == "GRAPHQL") self.assertLess(last_put, first_gql) + def test_rename_moves_likes_and_meta_and_records_the_rename(self): + likes = {"likes": 9, "delta_7d": 1, "rank": 1, "rank_delta_7d": 0, "flagged": False} + stale = {"to": "x/y", "at": iso(NOW - timedelta(days=8))} + recent = {"to": "c/d", "at": iso(NOW - timedelta(days=6))} + r, state = store({"discussions.json": {"alice/tool": 5}, "likes.json": {"alice/tool": likes}, + "meta.json": {"alice/tool": {"stars": 3}}, + "renames.json": {"old/gone": stale, "a/b": recent}}, threads({5: "alice/tool"})) + finish(FakeGitHub(r), self.plan([Action("rename", "alice/tool-ng", "", "alice/tool")]), now=NOW) + # The deploy after this run reads likes.json and meta.json under the new name. + self.assertEqual(state["likes.json"], {"alice/tool-ng": likes}) + self.assertEqual(state["meta.json"], {"alice/tool-ng": {"stars": 3}}) + # sync_likes reads old snapshots through this; past the 7-day baseline a record is dead weight. + self.assertEqual(state["renames.json"], {"a/b": recent, "alice/tool": {"to": "alice/tool-ng", "at": iso(NOW)}}) + def test_comments_on_each_entry_discussion(self): r = routes( discussions={"alice/tool": 5}, diff --git a/site/src/components/Leaderboard.astro b/site/src/components/Leaderboard.astro index 273456a..9446788 100644 --- a/site/src/components/Leaderboard.astro +++ b/site/src/components/Leaderboard.astro @@ -1,6 +1,6 @@ --- import type { Category, SiteEntry } from "../lib/data.ts"; -import { isFlagged, languages } from "../lib/data.ts"; +import { isFlagged, languages, trending } from "../lib/data.ts"; import { ago, compact, icon, num, pct } from "../lib/format.ts"; interface Props { @@ -14,6 +14,7 @@ const { entries, all, categories, current, label } = Astro.props; const counts = Object.fromEntries(categories.map((c) => [c.slug, all.filter((e) => e.category === c.slug).length])); const langs = languages(entries); const max = Math.max(1, ...entries.map((e) => e.likes)); +const hot = new Set(trending(all).map((e) => e.repo)); const now = Date.now(); const ts = (iso: string | null) => (iso ? Date.parse(iso) : 0); const split = (repo: string) => { @@ -110,7 +111,7 @@ const split = (repo: string) => { data-stars={e.stars ?? 0} data-pushed={ts(e.pushed_at)} data-new={ts(e.submitted_at)} - data-trend={!flagged && e.delta_7d > 0 ? "1" : undefined} + data-trend={hot.has(e.repo) ? "1" : undefined} data-lang={e.language ?? ""} data-q={`${e.repo} ${e.description}`.toLowerCase()} > diff --git a/site/tests/dist.test.mjs b/site/tests/dist.test.mjs index e2aec01..e42fb9b 100644 --- a/site/tests/dist.test.mjs +++ b/site/tests/dist.test.mjs @@ -125,3 +125,24 @@ test("hostile stats are neutralised in the HTML and the SVG", () => { } } }); + +test("the Trending tab marks exactly the rows trending() allows", async () => { + const { loadAll, trending } = await import("../src/lib/data.ts"); + const want = trending( + loadAll({ dataDir: fixtures, statsDir: join(fixtures, "stats") }).entries, + ).map((e) => e.repo.toLowerCase()); + for (const page of ["index.html", "category/cli-terminal/index.html"]) { + const rows = read(page).match(/]*data-q=[^>]*>/g) ?? []; + const marked = rows + .filter((r) => /\bdata-trend\b/.test(r)) + .map((r) => r.match(/data-q="([^ "]+)/)[1]); + assert.ok(rows.length > 0, page); + // zz-tie/same-likes climbs by 3, under TRENDING_MIN_DELTA, so the hero would skip it. + assert.ok(!marked.includes("zz-tie/same-likes"), page); + assert.deepEqual( + marked.sort(), + want.filter((r) => rows.some((x) => x.includes(`data-q="${r} `))).sort(), + page, + ); + } +}); diff --git a/site/tests/fixtures/stats/likes.json b/site/tests/fixtures/stats/likes.json index 1debac4..4ff4725 100644 --- a/site/tests/fixtures/stats/likes.json +++ b/site/tests/fixtures/stats/likes.json @@ -8,7 +8,7 @@ }, "zz-tie/same-likes": { "likes": 120, - "delta_7d": 0, + "delta_7d": 3, "rank": 3, "rank_delta_7d": 0, "flagged": false