Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
name: Deploy site

# Builds the leaderboard from main plus the stats branch and ships it to
# Cloudflare Pages, after every merge, Likes sync and site change.
# Cloudflare Pages, after every merge, Likes sync and site change, and again once
# a submission or health check has made its threads and moved its stats.
on:
push:
branches: [main]
paths: ["site/**", "data/**"]
workflow_run:
workflows: ["Create entry Discussions", "Sync Likes"]
workflows:
["Create entry Discussions", "Sync Likes", "Submit", "Health check"]
types: [completed]
workflow_dispatch:

Expand Down
11 changes: 5 additions & 6 deletions .github/workflows/on-merge.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
name: Create entry Discussions

# Every listed entry gets a Discussion; a thumbs-up on it is a Like. The
# hourly run picks up whatever the 40-per-run cap left behind.
# Every listed entry gets a Discussion; a thumbs-up on it is a Like. Submit and
# the health check make threads themselves, right after their PR merges; this
# hourly run is the backstop for the 40-per-run cap, failed creates and any
# other change to the list.
on:
push:
branches: [main]
paths: [data/entries.json]
schedule:
- cron: "23 * * * *"
workflow_dispatch:
Expand All @@ -14,7 +13,7 @@ permissions: {}

# Shared with rot.yml so this waits until a rename's keys have moved and its
# thread is retitled; otherwise it would open a second thread for the new name.
# A run dropped from the queue is picked up by the hourly schedule.
# A run dropped from the queue costs nothing: the next one covers the same list.
concurrency:
group: list-writes
cancel-in-progress: false
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/rot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,3 +92,8 @@ jobs:
env:
GITHUB_TOKEN: ${{ steps.token.outputs.token }}
run: python -m scripts.rot finish --plan "$RUNNER_TEMP/plan.json"
# Only after finish has moved a renamed entry's thread to its new key, or this would open a second one.
- if: steps.plan.outputs.changed == 'true'
env:
GITHUB_TOKEN: ${{ steps.token.outputs.token }}
run: python -m scripts.on_merge
16 changes: 16 additions & 0 deletions .github/workflows/submit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,22 @@ jobs:
python -m scripts.submit comment --result "$RESULT" --kind accepted > "$RUNNER_TEMP/comment.md"
gh issue comment "$ISSUE" --body-file "$RUNNER_TEMP/comment.md"
gh issue edit "$ISSUE" --add-label accepted
# The new entry's Like thread, made while this run still holds list-writes so a burst of
# submissions can't starve it. A failure is left to the hourly on-merge run, not parked.
- id: discussions_token
if: steps.apply.outputs.branch != ''
continue-on-error: true
uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
with:
app-id: ${{ secrets.APP_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
permission-contents: write
permission-discussions: write
- if: steps.apply.outputs.branch != '' && steps.discussions_token.outputs.token != ''
continue-on-error: true
env:
GITHUB_TOKEN: ${{ steps.discussions_token.outputs.token }}
run: python -m scripts.on_merge
- if: steps.apply.outputs.branch == ''
env:
GH_TOKEN: ${{ steps.issues_token.outputs.token }}
Expand Down
14 changes: 13 additions & 1 deletion scripts/rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
CONFIRM = timedelta(days=6)
# An older sighting is a different episode, so it starts the week of grace again.
STALE = timedelta(days=14)
RENAME_KEEP = timedelta(days=8)
REMOVAL_REASONS = (MSG_GONE, MSG_LICENSE, MSG_NO_README, MSG_SHORT_README)
REPO_INPUT = re.compile(r"[A-Za-z0-9-]{1,39}/[A-Za-z0-9._-]{1,100}")

Expand Down Expand Up @@ -245,7 +246,7 @@ def _move(data: dict, moves: dict[str, str]) -> dict:
return out


def finish(gh, plan: dict) -> None:
def finish(gh, plan: dict, now: datetime | None = None) -> None:
"""Write stats, then comment and retitle. Run only once the entries PR has merged.

A failed stats write doesn't hold back the comments; the run fails at the end instead.
Expand All @@ -261,13 +262,24 @@ def rekey(data):
data.pop(k, None)
return _move(data, moves)

now = now or datetime.now(timezone.utc)

def record(data):
# sync_likes replays these over snapshots taken before the rename; past its 7-day baseline they're unused.
kept = {k: v for k, v in data.items() if isinstance(v, dict) and now - _parse(v["at"]) < RENAME_KEEP}
return {**kept, **{old: {"to": new, "at": _iso(now)} for old, new in moves.items()}}

numbers = dict(read_json(gh, "discussions.json")[0] or {}) # before the move, so dropped entries find their thread
failed = 0
moved = True
for path, mutate, message in (
("rot.json", lambda _: plan["rot_state"], "chore(stats): record failing repos"),
("discussions.json", rekey, "chore(stats): follow renamed repos"),
("flags.json", rekey, "chore(stats): follow renamed repos"),
# The site build reads these two by key, so they move now rather than at the next hourly sync.
("likes.json", rekey, "chore(stats): follow renamed repos"),
("meta.json", rekey, "chore(stats): follow renamed repos"),
("renames.json", record, "chore(stats): record renamed repos"),
):
try:
update_json(gh, path, mutate, message)
Expand Down
13 changes: 10 additions & 3 deletions scripts/sync_likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,11 @@ def _snapshot_date(name: str):
return None


def _baseline(gh, now: datetime) -> dict | None:
"""The snapshot from 7 days ago, else the oldest one between 1 and 7 days old, else None."""
def _baseline(gh, now: datetime, renames: dict) -> dict | None:
"""The snapshot from 7 days ago, else the oldest one between 1 and 7 days old, else None.

Keys are moved through renames.json, so a renamed entry is compared with its own past.
"""
week, day = (now - WEEK).date(), (now - DAY).date()
dates = sorted(d for n in list_dir(gh, "snapshots") if (d := _snapshot_date(n)))
old = [d for d in dates if week < d <= day]
Expand All @@ -132,6 +135,10 @@ def _baseline(gh, now: datetime) -> dict | None:
if data is None:
# Listed but unreadable: fail closed rather than read it as an empty baseline.
raise GitHubError(404, f"snapshot {pick} is listed but could not be read")
# Oldest first, so a->b then b->c lands on c. A rename before the snapshot is already in it.
for old, r in sorted(renames.items(), key=lambda kv: kv[1]["at"]):
if _parse(r["at"]).date() >= pick and old in data:
data[r["to"]] = data.pop(old)
return data


Expand Down Expand Up @@ -219,7 +226,7 @@ def trusted(login):
candidates.sort(key=lambda c: c[0] != "burst")

ranks = rank_entries(likes, {k: m["stars"] for k, m in meta.items()})
old = _baseline(gh, now)
old = _baseline(gh, now, read_json(gh, "renames.json")[0] or {})
out = {}
for k in keys:
o = (old or {}).get(k)
Expand Down
46 changes: 46 additions & 0 deletions scripts/tests/test_likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,52 @@ def test_snapshot_written_once_per_day(self):
self.assertEqual(w2.written("likes.json")["o/a"]["likes"], 2)


class Renames(unittest.TestCase):
def snap(self, data):
return {f"snapshots/{WEEK_AGO}.json": data}

def test_baseline_follows_a_rename(self):
w = World([entry("o/new")], {"o/new": [node("x"), node("y"), node("z"), node("w")]},
stats={**self.snap({"o/old": {"likes": 3, "rank": 2}}),
"renames.json": {"o/old": {"to": "o/new", "at": iso(NOW - timedelta(days=2))}}})
w.run()
got = w.written("likes.json")["o/new"]
self.assertEqual((got["delta_7d"], got["rank_delta_7d"]), (1, 1))

def test_chained_renames_replay_in_order(self):
w = World([entry("o/c")], {"o/c": [node("x"), node("y")]},
stats={**self.snap({"o/a": {"likes": 1, "rank": 1}}),
"renames.json": {"o/b": {"to": "o/c", "at": iso(NOW - timedelta(days=1))},
"o/a": {"to": "o/b", "at": iso(NOW - timedelta(days=3))}}})
w.run()
self.assertEqual(w.written("likes.json")["o/c"]["delta_7d"], 1)

def test_rename_older_than_the_baseline_is_not_replayed(self):
# The snapshot already holds the new name, and the old one now belongs to someone else.
w = World([entry("o/a"), entry("o/new")], {"o/a": [node("x")], "o/new": [node("x"), node("y")]},
stats={**self.snap({"o/a": {"likes": 0, "rank": 2}, "o/new": {"likes": 2, "rank": 1}}),
"renames.json": {"o/a": {"to": "o/new", "at": iso(NOW - timedelta(days=9))}}})
w.run()
likes = w.written("likes.json")
self.assertEqual((likes["o/a"]["delta_7d"], likes["o/new"]["delta_7d"]), (1, 0))

def test_rename_then_sync_keeps_likes_and_trend(self):
from scripts.rot import Action, finish
from scripts.tests.test_rot import store, threads

row = {"likes": 3, "delta_7d": 0, "rank": 1, "rank_delta_7d": 0, "flagged": False}
r, state = store({"discussions.json": {"o/old": 1}, "likes.json": {"o/old": row}}, threads({1: "o/old"}))
plan = {"changed": True, "actions": [Action("rename", "o/new", "", "o/old").__dict__], "rot_state": {}}
finish(FakeGitHub(r), plan, now=NOW - timedelta(days=2))
self.assertEqual(state["likes.json"], {"o/new": row})

w = World([entry("o/new")], {"o/new": [node("x"), node("y"), node("z"), node("w")]},
stats={**state, **self.snap({"o/old": {"likes": 3, "rank": 1}})})
w.run()
got = w.written("likes.json")["o/new"]
self.assertEqual((got["likes"], got["delta_7d"], got["rank_delta_7d"]), (4, 1, 0))


class Writes(unittest.TestCase):
def test_write_order_and_commit_shape(self):
w = World([entry("o/a")], {"o/a": [node("x")]})
Expand Down
39 changes: 39 additions & 0 deletions scripts/tests/test_on_merge.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import base64
import json
import unittest
from pathlib import Path

from scripts.devstream.gh import GitHubError
from scripts.on_merge import CAP, run
Expand Down Expand Up @@ -149,5 +150,43 @@ def test_no_put_when_unchanged(self):
self.assertEqual(puts(g), [])


WORKFLOWS = Path(__file__).resolve().parents[2] / ".github" / "workflows"


def step_after(text, marker):
"""The text from marker to the end of the file."""
return text[text.index(marker):]


class Workflows(unittest.TestCase):
"""Threads are made by the run that changed the list, while it still holds list-writes."""

def test_rot_makes_threads_after_its_keys_move(self):
text = (WORKFLOWS / "rot.yml").read_text()
self.assertIn("python -m scripts.on_merge", step_after(text, "python -m scripts.rot finish"))

def test_submit_makes_the_thread_once_its_pr_has_merged(self):
text = (WORKFLOWS / "submit.yml").read_text()
apply = text[text.index("\n apply:\n"):text.index("\n reject:\n")]
tail = step_after(apply, "did not merge within 15 minutes")
self.assertIn("python -m scripts.on_merge", tail)
# A failed create is the hourly run's to retry; it mustn't park an accepted issue as needs-review.
step = tail[tail.rindex("- ", 0, tail.index("python -m scripts.on_merge")):]
self.assertIn("continue-on-error: true", step[:step.index("python -m scripts.on_merge")])

def test_on_merge_is_only_the_hourly_backstop(self):
text = (WORKFLOWS / "on-merge.yml").read_text()
on = text[text.index("\non:\n"):text.index("\npermissions:")]
self.assertNotIn("push:", on)
self.assertIn("schedule:", on)
self.assertIn("group: list-writes", text)

def test_site_redeploys_after_the_runs_that_make_threads(self):
text = (WORKFLOWS / "deploy.yml").read_text()
after = text[text.index("workflow_run:"):text.index("types: [completed]")]
for name in ("Submit", "Health check", "Create entry Discussions", "Sync Likes"):
self.assertIn(f'"{name}"', after)


if __name__ == "__main__":
unittest.main()
17 changes: 16 additions & 1 deletion scripts/tests/test_rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,7 @@ def file(data):
f"PUT {API}/contents/rot.json": None,
f"PUT {API}/contents/discussions.json": None,
f"PUT {API}/contents/flags.json": None,
f"PUT {API}/contents/renames.json": None,
f"GET {API}/git/ref/heads/stats": {"ref": "refs/heads/stats"},
}
r.update(extra or {})
Expand Down Expand Up @@ -241,7 +242,7 @@ def answer(body):
return answer

r = {f"GET {API}/git/ref/heads/stats": {"ref": "refs/heads/stats"}}
for path in ("rot.json", "discussions.json", "flags.json"):
for path in ("rot.json", "discussions.json", "flags.json", "likes.json", "meta.json", "renames.json"):
r[f"GET {API}/contents/{path}"] = get(path)
r[f"PUT {API}/contents/{path}"] = put(path)
r.update(extra or {})
Expand Down Expand Up @@ -483,6 +484,20 @@ def test_rename_moves_stats_keys(self):
first_gql = min(i for i, c in enumerate(g.calls) if c[0] == "GRAPHQL")
self.assertLess(last_put, first_gql)

def test_rename_moves_likes_and_meta_and_records_the_rename(self):
likes = {"likes": 9, "delta_7d": 1, "rank": 1, "rank_delta_7d": 0, "flagged": False}
stale = {"to": "x/y", "at": iso(NOW - timedelta(days=8))}
recent = {"to": "c/d", "at": iso(NOW - timedelta(days=6))}
r, state = store({"discussions.json": {"alice/tool": 5}, "likes.json": {"alice/tool": likes},
"meta.json": {"alice/tool": {"stars": 3}},
"renames.json": {"old/gone": stale, "a/b": recent}}, threads({5: "alice/tool"}))
finish(FakeGitHub(r), self.plan([Action("rename", "alice/tool-ng", "", "alice/tool")]), now=NOW)
# The deploy after this run reads likes.json and meta.json under the new name.
self.assertEqual(state["likes.json"], {"alice/tool-ng": likes})
self.assertEqual(state["meta.json"], {"alice/tool-ng": {"stars": 3}})
# sync_likes reads old snapshots through this; past the 7-day baseline a record is dead weight.
self.assertEqual(state["renames.json"], {"a/b": recent, "alice/tool": {"to": "alice/tool-ng", "at": iso(NOW)}})

def test_comments_on_each_entry_discussion(self):
r = routes(
discussions={"alice/tool": 5},
Expand Down
5 changes: 3 additions & 2 deletions site/src/components/Leaderboard.astro
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
import type { Category, SiteEntry } from "../lib/data.ts";
import { isFlagged, languages } from "../lib/data.ts";
import { isFlagged, languages, trending } from "../lib/data.ts";
import { ago, compact, icon, num, pct } from "../lib/format.ts";

interface Props {
Expand All @@ -14,6 +14,7 @@ const { entries, all, categories, current, label } = Astro.props;
const counts = Object.fromEntries(categories.map((c) => [c.slug, all.filter((e) => e.category === c.slug).length]));
const langs = languages(entries);
const max = Math.max(1, ...entries.map((e) => e.likes));
const hot = new Set(trending(all).map((e) => e.repo));
const now = Date.now();
const ts = (iso: string | null) => (iso ? Date.parse(iso) : 0);
const split = (repo: string) => {
Expand Down Expand Up @@ -110,7 +111,7 @@ const split = (repo: string) => {
data-stars={e.stars ?? 0}
data-pushed={ts(e.pushed_at)}
data-new={ts(e.submitted_at)}
data-trend={!flagged && e.delta_7d > 0 ? "1" : undefined}
data-trend={hot.has(e.repo) ? "1" : undefined}
data-lang={e.language ?? ""}
data-q={`${e.repo} ${e.description}`.toLowerCase()}
>
Expand Down
21 changes: 21 additions & 0 deletions site/tests/dist.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -125,3 +125,24 @@ test("hostile stats are neutralised in the HTML and the SVG", () => {
}
}
});

test("the Trending tab marks exactly the rows trending() allows", async () => {
const { loadAll, trending } = await import("../src/lib/data.ts");
const want = trending(
loadAll({ dataDir: fixtures, statsDir: join(fixtures, "stats") }).entries,
).map((e) => e.repo.toLowerCase());
for (const page of ["index.html", "category/cli-terminal/index.html"]) {
const rows = read(page).match(/<tr\b[^>]*data-q=[^>]*>/g) ?? [];
const marked = rows
.filter((r) => /\bdata-trend\b/.test(r))
.map((r) => r.match(/data-q="([^ "]+)/)[1]);
assert.ok(rows.length > 0, page);
// zz-tie/same-likes climbs by 3, under TRENDING_MIN_DELTA, so the hero would skip it.
assert.ok(!marked.includes("zz-tie/same-likes"), page);
assert.deepEqual(
marked.sort(),
want.filter((r) => rows.some((x) => x.includes(`data-q="${r} `))).sort(),
page,
);
}
});
2 changes: 1 addition & 1 deletion site/tests/fixtures/stats/likes.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
},
"zz-tie/same-likes": {
"likes": 120,
"delta_7d": 0,
"delta_7d": 3,
"rank": 3,
"rank_delta_7d": 0,
"flagged": false
Expand Down
Loading