diff --git a/scripts/sweep.py b/scripts/sweep.py index 51e80d0..598f03d 100644 --- a/scripts/sweep.py +++ b/scripts/sweep.py @@ -24,6 +24,8 @@ ROT_BRANCH = re.compile(r"rot/[0-9]{1,20}") # rot.yml's run-name for a full run; a one-repo re-check ("Health check: owner/name") doesn't cover the list. FULL_ROT = "Health check" +# Both are hourly; GitHub skips scheduled runs under load, so a sweep that runs restarts a stalled one. +HOURLY_STALE = timedelta(hours=2) def _parse(s: str) -> datetime: @@ -107,6 +109,19 @@ def _rot_overdue(gh, now) -> bool: return latest is None or now - latest > ROT_RETRY +def _stale(gh, wf, now) -> bool: + page = gh.get(f"{API}/actions/workflows/{wf}/runs?per_page=1")["workflow_runs"] + return not page or now - _parse(page[0]["created_at"]) > HOURLY_STALE + + +def _dispatch_if_stale(gh, wf, now) -> bool: + if not _stale(gh, wf, now): + return False + gh.post(f"{API}/actions/workflows/{wf}/dispatches", {"ref": "main"}) + print(f"dispatched {wf}: no run in {HOURLY_STALE}") + return True + + def _list_write_waiting(gh) -> bool: # These share the list-writes group, which holds one waiting run. return any( @@ -136,6 +151,8 @@ def sweep(gh, now: datetime, entries=()) -> None: busy = _sweep_branches(gh, now) _sweep_rot_branches(gh, now) _close_rot_prs(gh, now) + # Likes has its own concurrency group, so it never waits on list-writes. + _dispatch_if_stale(gh, "likes.yml", now) # The list-writes group keeps one pending run and cancels older ones, so never queue behind a waiting run. if _list_write_waiting(gh): print("a list-writes run is already waiting") @@ -165,6 +182,8 @@ def sweep(gh, now: datetime, entries=()) -> None: continue todo.append((issue, workflow)) if not todo: + # Nothing else wants the list-writes slot, so a stalled hourly on-merge can have it. + _dispatch_if_stale(gh, "on-merge.yml", now) return oldest, workflow = min(todo, key=lambda t: _parse(t[0]["created_at"])) gh.post(f"{API}/actions/workflows/{workflow}/dispatches", diff --git a/scripts/tests/test_sweep.py b/scripts/tests/test_sweep.py index 97a3d25..56e95a0 100644 --- a/scripts/tests/test_sweep.py +++ b/scripts/tests/test_sweep.py @@ -24,6 +24,10 @@ ROT_DISPATCH = f"POST {API}/actions/workflows/rot.yml/dispatches" ROT_REFS = f"GET {API}/git/matching-refs/heads/rot/" OPEN_PRS = f"GET {API}/pulls?state=open&per_page=100" +LIKES_RUNS = f"GET {API}/actions/workflows/likes.yml/runs?per_page=1" +MERGE_RUNS = f"GET {API}/actions/workflows/on-merge.yml/runs?per_page=1" +LIKES_DISPATCH = f"POST {API}/actions/workflows/likes.yml/dispatches" +MERGE_DISPATCH = f"POST {API}/actions/workflows/on-merge.yml/dispatches" BOT = "devstreamhq-bot[bot]" @@ -64,6 +68,8 @@ def gh(issues=(), refs=(), routes=None): REPORT_QUEUED: {"total_count": 0}, REPORT_PENDING: {"total_count": 0}, REPORT_DISPATCH: None, ROT_OK: runs(timedelta(days=2)), ROT_LATEST: runs(timedelta(days=2)), ROT_DISPATCH: None, ROT_REFS: [], OPEN_PRS: [], + LIKES_RUNS: runs(timedelta(minutes=20)), MERGE_RUNS: runs(timedelta(minutes=20)), + LIKES_DISPATCH: None, MERGE_DISPATCH: None, } r.update(routes or {}) return FakeGitHub(r) @@ -89,6 +95,10 @@ def rot_dispatched(g): return [b for m, p, b in g.calls if m == "POST" and "rot.yml" in p] +def hourly_dispatched(g): + return [p.split("/")[-2] for m, p, b in g.calls if m == "POST" and ("likes.yml" in p or "on-merge.yml" in p)] + + def closed(g): return [(p, b) for m, p, b in g.calls if m == "PATCH"] @@ -355,3 +365,36 @@ def test_ignores_refs_that_are_not_submit_numbers(self): if __name__ == "__main__": unittest.main() + + +class HourlyWatchdogTest(unittest.TestCase): + # GitHub skips scheduled runs under load; a sweep that does run restarts a stalled hourly job. + def test_fresh_hourly_runs_are_left_alone(self): + g = gh() + sweep(g, NOW) + self.assertEqual(hourly_dispatched(g), []) + + def test_stale_likes_is_dispatched_even_when_list_writes_is_busy(self): + g = gh(routes={LIKES_RUNS: runs(timedelta(hours=3)), MERGE_QUEUED: {"total_count": 1}}) + sweep(g, NOW) + self.assertEqual(hourly_dispatched(g), ["likes.yml"]) + + def test_likes_with_no_runs_at_all_is_dispatched(self): + g = gh(routes={LIKES_RUNS: {"total_count": 0, "workflow_runs": []}}) + sweep(g, NOW) + self.assertEqual(hourly_dispatched(g), ["likes.yml"]) + + def test_stale_on_merge_waits_for_a_free_list_writes_slot(self): + g = gh(routes={MERGE_RUNS: runs(timedelta(hours=3)), QUEUED: {"total_count": 1}}) + sweep(g, NOW) + self.assertEqual(hourly_dispatched(g), []) + + def test_stale_on_merge_is_dispatched_when_nothing_else_is(self): + g = gh(routes={MERGE_RUNS: runs(timedelta(hours=3))}) + sweep(g, NOW) + self.assertEqual(hourly_dispatched(g), ["on-merge.yml"]) + + def test_a_pending_submission_goes_before_a_stale_on_merge(self): + g = gh([issue(9)], routes={MERGE_RUNS: runs(timedelta(hours=3))}) + sweep(g, NOW) + self.assertEqual((dispatched(g), hourly_dispatched(g)), (["9"], []))