Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .github/workflows/likes.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@ name: Sync Likes
# Counts trusted thumbs-ups on each entry's Discussion, ranks entries, flags
# Like bursts and writes likes.json, meta.json and daily snapshots to stats.
on:
schedule:
- cron: "17 * * * *"
# Started on time by the scheduler Worker (scheduler/); a stalled one is restarted by the sweep.
workflow_dispatch:

permissions: {}
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/on-merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ name: Create entry Discussions
# hourly run is the backstop for the 40-per-run cap, failed creates and any
# other change to the list.
on:
schedule:
- cron: "23 * * * *"
# Started on time by the scheduler Worker (scheduler/); a stalled one is restarted by the sweep.
workflow_dispatch:

permissions: {}
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/rot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ run-name: "${{ inputs.repo && format('Health check: {0}', inputs.repo) || 'Healt
# days later, brings back the ones that recover, follows renames and removes
# repos that are gone (only once a second run a week later agrees).
on:
schedule:
- cron: "0 6 * * 1"
# Started on time by the scheduler Worker (scheduler/); a stalled one is restarted by the sweep.
workflow_dispatch:
inputs:
repo:
Expand Down
44 changes: 44 additions & 0 deletions scheduler/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
# Scheduler

GitHub Actions skips scheduled runs when it's busy, and on a quiet repo it skips a lot of them. Likes once went seven hours without a sync. This Cloudflare Worker starts the timed workflows instead, through the GitHub API, on a cron that Cloudflare keeps.

One cron trigger fires every 15 minutes. Each tick starts:

| When (UTC) | Workflow |
| -------------- | ----------------------------------- |
| every tick | `sweep.yml` |
| :15 every hour | `likes.yml` |
| :30 every hour | `on-merge.yml` |
| Mondays 06:45 | `rot.yml` (the weekly health check) |
| daily 09:45 | `stars.yml` |

`likes.yml`, `on-merge.yml` and `rot.yml` have no GitHub schedule of their own, so they never run twice. `sweep.yml` and `stars.yml` keep theirs as a fallback. If the Worker stops, the sweep still restarts a stalled Likes sync or on-merge run, and still runs an overdue health check.

The Worker has no URL. It does nothing but run on its cron.

## Token

The Worker needs one secret, `GITHUB_TOKEN`: a fine-grained personal access token with

- resource owner `devstreamhq`, repository `devstreamhq/devstream` only
- **Actions: Read and write**, nothing else
- an expiry about a year out (it's tracked with the other rotations)

## Deploy

From the repo root, with the Wrangler version the site pins:

```sh
cd scheduler
npx --prefix ../site wrangler login
npx --prefix ../site wrangler deploy
npx --prefix ../site wrangler secret put GITHUB_TOKEN
```

The last command asks you to paste the token. Run `wrangler deploy` again whenever `index.js` or `wrangler.toml` changes. The secret stays put.

To check it's working, open the Worker in the Cloudflare dashboard and look at **Cron events**, or watch the Actions tab for `workflow_dispatch` runs every 15 minutes.

## Tests

`site/tests/scheduler.test.mjs` runs with the site tests (`npm --prefix site test`).
54 changes: 54 additions & 0 deletions scheduler/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
// Starts DevStream's timed GitHub workflows. GitHub's own scheduler skips runs
// under load (Likes once went seven hours without one); Cloudflare's doesn't.

const API =
"https://api.github.com/repos/devstreamhq/devstream/actions/workflows";

// One cron trigger, every 15 minutes; the tick's time picks what else runs.
export function workflowsFor(when) {
const m = when.getUTCMinutes();
const h = when.getUTCHours();
const out = ["sweep.yml"];
if (m === 15) out.push("likes.yml");
if (m === 30) out.push("on-merge.yml");
// Monday; kept off :30 so it never shares the list-writes queue with on-merge.
if (when.getUTCDay() === 1 && h === 6 && m === 45) out.push("rot.yml");
if (h === 9 && m === 45) out.push("stars.yml");
return out;
}

export async function dispatch(workflow, token, fetchFn = fetch) {
const res = await fetchFn(`${API}/${workflow}/dispatches`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
Accept: "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "devstream-scheduler",
},
body: JSON.stringify({ ref: "main" }),
});
if (res.status !== 204) {
throw new Error(`${workflow}: GitHub answered ${res.status}`);
}
}

export async function run(when, token, fetchFn = fetch) {
// Every dispatch is tried; one failure mustn't stop the rest.
const results = await Promise.allSettled(
workflowsFor(when).map((wf) => dispatch(wf, token, fetchFn)),
);
const failed = results.filter((r) => r.status === "rejected");
if (failed.length) {
// Thrown so the Cron Trigger shows as failed in the Cloudflare dashboard.
throw new Error(failed.map((r) => r.reason.message).join("; "));
}
}

export default {
async scheduled(controller, env) {
// scheduledTime is the planned slot, not when the tick actually ran; drop any seconds.
const when = new Date(Math.floor(controller.scheduledTime / 60000) * 60000);
await run(when, env.GITHUB_TOKEN);
},
};
9 changes: 9 additions & 0 deletions scheduler/wrangler.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
name = "devstream-scheduler"
main = "index.js"
compatibility_date = "2026-10-01"
# No public URL: the Worker only runs on its cron.
workers_dev = false
preview_urls = false

[triggers]
crons = ["*/15 * * * *"]
3 changes: 2 additions & 1 deletion scripts/admin/setup_repo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ RULESET=$(cat <<'J'
{ "context": "lint", "integration_id": 15368 },
{ "context": "analyze (python)", "integration_id": 15368 },
{ "context": "analyze (javascript-typescript)", "integration_id": 15368 },
{ "context": "analyze (actions)", "integration_id": 15368 }
{ "context": "analyze (actions)", "integration_id": 15368 },
{ "context": "site", "integration_id": 15368 }
]
}
},
Expand Down
4 changes: 3 additions & 1 deletion scripts/tests/test_on_merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,9 @@ def test_on_merge_is_only_the_hourly_backstop(self):
text = (WORKFLOWS / "on-merge.yml").read_text()
on = text[text.index("\non:\n"):text.index("\npermissions:")]
self.assertNotIn("push:", on)
self.assertIn("schedule:", on)
# Started hourly by the scheduler Worker; a GitHub schedule too would queue a second run in list-writes.
self.assertNotIn("schedule:", on)
self.assertIn("workflow_dispatch:", on)
self.assertIn("group: list-writes", text)

def test_site_redeploys_after_the_runs_that_make_threads(self):
Expand Down
106 changes: 106 additions & 0 deletions site/tests/scheduler.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
import { test } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";

import worker, { dispatch, run, workflowsFor } from "../../scheduler/index.js";

const read = (p) => readFileSync(new URL(p, import.meta.url), "utf8");
const at = (iso) => new Date(iso);

function fakeFetch(statuses = {}) {
const calls = [];
const fn = async (url, init) => {
calls.push({ url, init });
const wf = url.split("/").at(-2);
return { status: statuses[wf] ?? 204 };
};
return { fn, calls };
}

test("every tick starts the sweep, and only the sweep off the hourly slots", () => {
assert.deepEqual(workflowsFor(at("2026-11-07T10:00:00Z")), ["sweep.yml"]);
assert.deepEqual(workflowsFor(at("2026-11-07T10:45:00Z")), ["sweep.yml"]);
});

test("likes at :15 and on-merge at :30, every hour", () => {
assert.deepEqual(workflowsFor(at("2026-11-07T03:15:00Z")), [
"sweep.yml",
"likes.yml",
]);
assert.deepEqual(workflowsFor(at("2026-11-07T03:30:00Z")), [
"sweep.yml",
"on-merge.yml",
]);
});

test("the health check runs Mondays at 06:45 only", () => {
// 2026-11-09 is a Monday.
assert.deepEqual(workflowsFor(at("2026-11-09T06:45:00Z")), [
"sweep.yml",
"rot.yml",
]);
assert.deepEqual(workflowsFor(at("2026-11-10T06:45:00Z")), ["sweep.yml"]);
assert.deepEqual(workflowsFor(at("2026-11-09T07:45:00Z")), ["sweep.yml"]);
});

test("stars runs daily at 09:45", () => {
assert.deepEqual(workflowsFor(at("2026-11-07T09:45:00Z")), [
"sweep.yml",
"stars.yml",
]);
});

test("a dispatch posts to main with the token", async () => {
const { fn, calls } = fakeFetch();
await dispatch("likes.yml", "tok", fn);
assert.equal(
calls[0].url,
"https://api.github.com/repos/devstreamhq/devstream/actions/workflows/likes.yml/dispatches",
);
assert.equal(calls[0].init.method, "POST");
assert.equal(calls[0].init.headers.Authorization, "Bearer tok");
assert.ok(calls[0].init.headers["User-Agent"]);
assert.deepEqual(JSON.parse(calls[0].init.body), { ref: "main" });
});

test("one failed dispatch still lets the others go, then fails the tick", async () => {
const { fn, calls } = fakeFetch({ "sweep.yml": 500 });
await assert.rejects(
run(at("2026-11-07T03:15:00Z"), "tok", fn),
/sweep\.yml: GitHub answered 500/,
);
assert.equal(calls.length, 2);
});

test("a late tick still counts as the slot it was scheduled for", async () => {
const { fn, calls } = fakeFetch();
const real = globalThis.fetch;
globalThis.fetch = fn;
try {
const slot = Date.parse("2026-11-07T03:15:00Z");
await worker.scheduled(
{ scheduledTime: slot + 20_000 },
{ GITHUB_TOKEN: "tok" },
);
} finally {
globalThis.fetch = real;
}
assert.deepEqual(
calls.map((c) => c.url.split("/").at(-2)),
["sweep.yml", "likes.yml"],
);
});

test("the worker's cron and the workflows' own schedules don't double up", () => {
assert.match(
read("../../scheduler/wrangler.toml"),
/crons = \["\*\/15 \* \* \* \*"\]/,
);
// The Worker owns these; a second, GitHub-scheduled run would queue behind it.
for (const wf of ["likes", "on-merge", "rot"]) {
assert.ok(
!read(`../../.github/workflows/${wf}.yml`).includes("schedule:"),
`${wf}.yml still has a schedule`,
);
}
});
Loading