From 1331f15f42b17e875187fc35ba3ae0890f786f2d Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:01:46 -0700 Subject: [PATCH] feat(scheduler): start timed workflows from a Cloudflare cron GitHub keeps skipping GitHub's scheduler skipped most runs today: Likes went seven hours without a sync. A Worker with one 15-minute cron now dispatches sweep, likes, on-merge, the weekly health check and stars. likes, on-merge and rot lose their GitHub schedules so nothing runs twice; sweep and stars keep theirs as a fallback, and the sweep watchdog still restarts stalled jobs if the Worker stops. setup_repo.sh also lists the new site check. --- .github/workflows/likes.yml | 3 +- .github/workflows/on-merge.yml | 3 +- .github/workflows/rot.yml | 3 +- scheduler/README.md | 44 ++++++++++++++ scheduler/index.js | 54 +++++++++++++++++ scheduler/wrangler.toml | 9 +++ scripts/admin/setup_repo.sh | 3 +- scripts/tests/test_on_merge.py | 4 +- site/tests/scheduler.test.mjs | 106 +++++++++++++++++++++++++++++++++ 9 files changed, 221 insertions(+), 8 deletions(-) create mode 100644 scheduler/README.md create mode 100644 scheduler/index.js create mode 100644 scheduler/wrangler.toml create mode 100644 site/tests/scheduler.test.mjs diff --git a/.github/workflows/likes.yml b/.github/workflows/likes.yml index fcac2ff..3581773 100644 --- a/.github/workflows/likes.yml +++ b/.github/workflows/likes.yml @@ -3,8 +3,7 @@ name: Sync Likes # Counts trusted thumbs-ups on each entry's Discussion, ranks entries, flags # Like bursts and writes likes.json, meta.json and daily snapshots to stats. on: - schedule: - - cron: "17 * * * *" + # Started on time by the scheduler Worker (scheduler/); a stalled one is restarted by the sweep. workflow_dispatch: permissions: {} diff --git a/.github/workflows/on-merge.yml b/.github/workflows/on-merge.yml index ae71d2d..f911483 100644 --- a/.github/workflows/on-merge.yml +++ b/.github/workflows/on-merge.yml @@ -5,8 +5,7 @@ name: Create entry Discussions # hourly run is the backstop for the 40-per-run cap, failed creates and any # other change to the list. on: - schedule: - - cron: "23 * * * *" + # Started on time by the scheduler Worker (scheduler/); a stalled one is restarted by the sweep. workflow_dispatch: permissions: {} diff --git a/.github/workflows/rot.yml b/.github/workflows/rot.yml index 8456ef3..891ef47 100644 --- a/.github/workflows/rot.yml +++ b/.github/workflows/rot.yml @@ -7,8 +7,7 @@ run-name: "${{ inputs.repo && format('Health check: {0}', inputs.repo) || 'Healt # days later, brings back the ones that recover, follows renames and removes # repos that are gone (only once a second run a week later agrees). on: - schedule: - - cron: "0 6 * * 1" + # Started on time by the scheduler Worker (scheduler/); a stalled one is restarted by the sweep. workflow_dispatch: inputs: repo: diff --git a/scheduler/README.md b/scheduler/README.md new file mode 100644 index 0000000..29be0f0 --- /dev/null +++ b/scheduler/README.md @@ -0,0 +1,44 @@ +# Scheduler + +GitHub Actions skips scheduled runs when it's busy, and on a quiet repo it skips a lot of them. Likes once went seven hours without a sync. This Cloudflare Worker starts the timed workflows instead, through the GitHub API, on a cron that Cloudflare keeps. + +One cron trigger fires every 15 minutes. Each tick starts: + +| When (UTC) | Workflow | +| -------------- | ----------------------------------- | +| every tick | `sweep.yml` | +| :15 every hour | `likes.yml` | +| :30 every hour | `on-merge.yml` | +| Mondays 06:45 | `rot.yml` (the weekly health check) | +| daily 09:45 | `stars.yml` | + +`likes.yml`, `on-merge.yml` and `rot.yml` have no GitHub schedule of their own, so they never run twice. `sweep.yml` and `stars.yml` keep theirs as a fallback. If the Worker stops, the sweep still restarts a stalled Likes sync or on-merge run, and still runs an overdue health check. + +The Worker has no URL. It does nothing but run on its cron. + +## Token + +The Worker needs one secret, `GITHUB_TOKEN`: a fine-grained personal access token with + +- resource owner `devstreamhq`, repository `devstreamhq/devstream` only +- **Actions: Read and write**, nothing else +- an expiry about a year out (it's tracked with the other rotations) + +## Deploy + +From the repo root, with the Wrangler version the site pins: + +```sh +cd scheduler +npx --prefix ../site wrangler login +npx --prefix ../site wrangler deploy +npx --prefix ../site wrangler secret put GITHUB_TOKEN +``` + +The last command asks you to paste the token. Run `wrangler deploy` again whenever `index.js` or `wrangler.toml` changes. The secret stays put. + +To check it's working, open the Worker in the Cloudflare dashboard and look at **Cron events**, or watch the Actions tab for `workflow_dispatch` runs every 15 minutes. + +## Tests + +`site/tests/scheduler.test.mjs` runs with the site tests (`npm --prefix site test`). diff --git a/scheduler/index.js b/scheduler/index.js new file mode 100644 index 0000000..369155a --- /dev/null +++ b/scheduler/index.js @@ -0,0 +1,54 @@ +// Starts DevStream's timed GitHub workflows. GitHub's own scheduler skips runs +// under load (Likes once went seven hours without one); Cloudflare's doesn't. + +const API = + "https://api.github.com/repos/devstreamhq/devstream/actions/workflows"; + +// One cron trigger, every 15 minutes; the tick's time picks what else runs. +export function workflowsFor(when) { + const m = when.getUTCMinutes(); + const h = when.getUTCHours(); + const out = ["sweep.yml"]; + if (m === 15) out.push("likes.yml"); + if (m === 30) out.push("on-merge.yml"); + // Monday; kept off :30 so it never shares the list-writes queue with on-merge. + if (when.getUTCDay() === 1 && h === 6 && m === 45) out.push("rot.yml"); + if (h === 9 && m === 45) out.push("stars.yml"); + return out; +} + +export async function dispatch(workflow, token, fetchFn = fetch) { + const res = await fetchFn(`${API}/${workflow}/dispatches`, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "devstream-scheduler", + }, + body: JSON.stringify({ ref: "main" }), + }); + if (res.status !== 204) { + throw new Error(`${workflow}: GitHub answered ${res.status}`); + } +} + +export async function run(when, token, fetchFn = fetch) { + // Every dispatch is tried; one failure mustn't stop the rest. + const results = await Promise.allSettled( + workflowsFor(when).map((wf) => dispatch(wf, token, fetchFn)), + ); + const failed = results.filter((r) => r.status === "rejected"); + if (failed.length) { + // Thrown so the Cron Trigger shows as failed in the Cloudflare dashboard. + throw new Error(failed.map((r) => r.reason.message).join("; ")); + } +} + +export default { + async scheduled(controller, env) { + // scheduledTime is the planned slot, not when the tick actually ran; drop any seconds. + const when = new Date(Math.floor(controller.scheduledTime / 60000) * 60000); + await run(when, env.GITHUB_TOKEN); + }, +}; diff --git a/scheduler/wrangler.toml b/scheduler/wrangler.toml new file mode 100644 index 0000000..7f78a04 --- /dev/null +++ b/scheduler/wrangler.toml @@ -0,0 +1,9 @@ +name = "devstream-scheduler" +main = "index.js" +compatibility_date = "2026-10-01" +# No public URL: the Worker only runs on its cron. +workers_dev = false +preview_urls = false + +[triggers] +crons = ["*/15 * * * *"] diff --git a/scripts/admin/setup_repo.sh b/scripts/admin/setup_repo.sh index f8c432a..2bc7142 100755 --- a/scripts/admin/setup_repo.sh +++ b/scripts/admin/setup_repo.sh @@ -64,7 +64,8 @@ RULESET=$(cat <<'J' { "context": "lint", "integration_id": 15368 }, { "context": "analyze (python)", "integration_id": 15368 }, { "context": "analyze (javascript-typescript)", "integration_id": 15368 }, - { "context": "analyze (actions)", "integration_id": 15368 } + { "context": "analyze (actions)", "integration_id": 15368 }, + { "context": "site", "integration_id": 15368 } ] } }, diff --git a/scripts/tests/test_on_merge.py b/scripts/tests/test_on_merge.py index 441826c..2162166 100644 --- a/scripts/tests/test_on_merge.py +++ b/scripts/tests/test_on_merge.py @@ -233,7 +233,9 @@ def test_on_merge_is_only_the_hourly_backstop(self): text = (WORKFLOWS / "on-merge.yml").read_text() on = text[text.index("\non:\n"):text.index("\npermissions:")] self.assertNotIn("push:", on) - self.assertIn("schedule:", on) + # Started hourly by the scheduler Worker; a GitHub schedule too would queue a second run in list-writes. + self.assertNotIn("schedule:", on) + self.assertIn("workflow_dispatch:", on) self.assertIn("group: list-writes", text) def test_site_redeploys_after_the_runs_that_make_threads(self): diff --git a/site/tests/scheduler.test.mjs b/site/tests/scheduler.test.mjs new file mode 100644 index 0000000..5fe3690 --- /dev/null +++ b/site/tests/scheduler.test.mjs @@ -0,0 +1,106 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +import worker, { dispatch, run, workflowsFor } from "../../scheduler/index.js"; + +const read = (p) => readFileSync(new URL(p, import.meta.url), "utf8"); +const at = (iso) => new Date(iso); + +function fakeFetch(statuses = {}) { + const calls = []; + const fn = async (url, init) => { + calls.push({ url, init }); + const wf = url.split("/").at(-2); + return { status: statuses[wf] ?? 204 }; + }; + return { fn, calls }; +} + +test("every tick starts the sweep, and only the sweep off the hourly slots", () => { + assert.deepEqual(workflowsFor(at("2026-11-07T10:00:00Z")), ["sweep.yml"]); + assert.deepEqual(workflowsFor(at("2026-11-07T10:45:00Z")), ["sweep.yml"]); +}); + +test("likes at :15 and on-merge at :30, every hour", () => { + assert.deepEqual(workflowsFor(at("2026-11-07T03:15:00Z")), [ + "sweep.yml", + "likes.yml", + ]); + assert.deepEqual(workflowsFor(at("2026-11-07T03:30:00Z")), [ + "sweep.yml", + "on-merge.yml", + ]); +}); + +test("the health check runs Mondays at 06:45 only", () => { + // 2026-11-09 is a Monday. + assert.deepEqual(workflowsFor(at("2026-11-09T06:45:00Z")), [ + "sweep.yml", + "rot.yml", + ]); + assert.deepEqual(workflowsFor(at("2026-11-10T06:45:00Z")), ["sweep.yml"]); + assert.deepEqual(workflowsFor(at("2026-11-09T07:45:00Z")), ["sweep.yml"]); +}); + +test("stars runs daily at 09:45", () => { + assert.deepEqual(workflowsFor(at("2026-11-07T09:45:00Z")), [ + "sweep.yml", + "stars.yml", + ]); +}); + +test("a dispatch posts to main with the token", async () => { + const { fn, calls } = fakeFetch(); + await dispatch("likes.yml", "tok", fn); + assert.equal( + calls[0].url, + "https://api.github.com/repos/devstreamhq/devstream/actions/workflows/likes.yml/dispatches", + ); + assert.equal(calls[0].init.method, "POST"); + assert.equal(calls[0].init.headers.Authorization, "Bearer tok"); + assert.ok(calls[0].init.headers["User-Agent"]); + assert.deepEqual(JSON.parse(calls[0].init.body), { ref: "main" }); +}); + +test("one failed dispatch still lets the others go, then fails the tick", async () => { + const { fn, calls } = fakeFetch({ "sweep.yml": 500 }); + await assert.rejects( + run(at("2026-11-07T03:15:00Z"), "tok", fn), + /sweep\.yml: GitHub answered 500/, + ); + assert.equal(calls.length, 2); +}); + +test("a late tick still counts as the slot it was scheduled for", async () => { + const { fn, calls } = fakeFetch(); + const real = globalThis.fetch; + globalThis.fetch = fn; + try { + const slot = Date.parse("2026-11-07T03:15:00Z"); + await worker.scheduled( + { scheduledTime: slot + 20_000 }, + { GITHUB_TOKEN: "tok" }, + ); + } finally { + globalThis.fetch = real; + } + assert.deepEqual( + calls.map((c) => c.url.split("/").at(-2)), + ["sweep.yml", "likes.yml"], + ); +}); + +test("the worker's cron and the workflows' own schedules don't double up", () => { + assert.match( + read("../../scheduler/wrangler.toml"), + /crons = \["\*\/15 \* \* \* \*"\]/, + ); + // The Worker owns these; a second, GitHub-scheduled run would queue behind it. + for (const wf of ["likes", "on-merge", "rot"]) { + assert.ok( + !read(`../../.github/workflows/${wf}.yml`).includes("schedule:"), + `${wf}.yml still has a schedule`, + ); + } +});