diff --git a/.github/workflows/submit.yml b/.github/workflows/submit.yml index 2a1b0de..8a71019 100644 --- a/.github/workflows/submit.yml +++ b/.github/workflows/submit.yml @@ -194,15 +194,20 @@ jobs: client-id: ${{ vars.APP_CLIENT_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} permission-issues: write + # The entry is live now, so a failure here must not park it as needs-review; the sweeper + # finishes the label and close if the issue is still open. The label goes on first and is + # retried; the comment is a create, so it is sent once and never retried. - if: steps.apply.outputs.branch != '' + continue-on-error: true env: GH_TOKEN: ${{ steps.late_issues_token.outputs.token || steps.issues_token.outputs.token }} RESULT: ${{ runner.temp }}/result/result.json ISSUE: ${{ steps.meta.outputs.issue }} run: | python -m scripts.submit comment --result "$RESULT" --kind accepted > "$RUNNER_TEMP/comment.md" + retry() { for i in 1 2 3; do "$@" && return; [ "$i" = 3 ] || sleep $((i * 10)); done; return 1; } + retry gh issue edit "$ISSUE" --add-label accepted gh issue comment "$ISSUE" --body-file "$RUNNER_TEMP/comment.md" - gh issue edit "$ISSUE" --add-label accepted # The new entry's Like thread, made while this run still holds list-writes so a burst of # submissions can't starve it. A failure is left to the hourly on-merge run, not parked. - id: discussions_token diff --git a/.github/workflows/sweep.yml b/.github/workflows/sweep.yml index fb2000d..82f787c 100644 --- a/.github/workflows/sweep.yml +++ b/.github/workflows/sweep.yml @@ -39,7 +39,7 @@ jobs: private-key: ${{ secrets.APP_PRIVATE_KEY }} permission-actions: write permission-contents: write - permission-issues: read + permission-issues: write permission-pull-requests: write - run: python -m scripts.sweep env: diff --git a/scripts/sweep.py b/scripts/sweep.py index 1964f1e..51e80d0 100644 --- a/scripts/sweep.py +++ b/scripts/sweep.py @@ -8,7 +8,9 @@ import re from datetime import datetime, timedelta, timezone +from scripts.devstream.entries import load from scripts.devstream.gh import GitHub, GitHubError +from scripts.submit import comment OWNER = "devstreamhq" API = f"/repos/{OWNER}/devstream" @@ -113,7 +115,23 @@ def _list_write_waiting(gh) -> bool: ) -def sweep(gh, now: datetime) -> None: +def _finish_listed(gh, issue, entry) -> None: + """The issue's entry is on the list, so its run merged; post what a failed follow-up step didn't, and close it.""" + n = issue["number"] + try: + if "accepted" not in {lb["name"] for lb in issue["labels"]}: + # Label first: a comment is a create, sent at most once, and the label marks it as tried. + gh.post(f"{API}/issues/{n}/labels", {"labels": ["accepted"]}) + gh.post(f"{API}/issues/{n}/comments", {"body": comment({"entry": entry}, "accepted")}) + gh.patch(f"{API}/issues/{n}", {"state": "closed", "state_reason": "completed"}) + except GitHubError as e: + # Tried again next sweep; one bad issue mustn't stop the dispatches. + print(f"could not close #{n}: {e}") + return + print(f"closed #{n}: already on the list") + + +def sweep(gh, now: datetime, entries=()) -> None: # Branches first, so an issue freed by a deleted branch can be dispatched this run. busy = _sweep_branches(gh, now) _sweep_rot_branches(gh, now) @@ -128,12 +146,18 @@ def sweep(gh, now: datetime) -> None: print("dispatched rot.yml") return todo = [] + # Older than the longest run (merge wait included), so the run that added it has finished. + listed = {e["issue"]: e for e in entries if now - _parse(e["submitted_at"]) > BRANCH_AGE} # Submissions and reports share the one dispatch; every handled report is closed or needs-review. for label, workflow, done in (("submission", "submit.yml", {"accepted", "rejected", "needs-review"}), ("report", "reports.yml", {"needs-review"})): for issue in gh.paginate(f"{API}/issues?state=open&labels={label}&per_page=100"): labels = {lb["name"] for lb in issue["labels"]} user = issue.get("user") or {} + # Before the busy check: a merged branch that won't delete must not keep its issue open. + if label == "submission" and label in labels and issue["number"] in listed and "pull_request" not in issue: + _finish_listed(gh, issue, listed[issue["number"]]) + continue if (label not in labels or "pull_request" in issue or labels & done or issue["number"] in busy or now - _parse(issue["created_at"]) <= ISSUE_AGE # The bot's own Like-burst reports are for a maintainer, not reports.yml. @@ -149,4 +173,4 @@ def sweep(gh, now: datetime) -> None: if __name__ == "__main__": - sweep(GitHub(os.environ["GITHUB_TOKEN"]), datetime.now(timezone.utc)) + sweep(GitHub(os.environ["GITHUB_TOKEN"]), datetime.now(timezone.utc), load()) diff --git a/scripts/tests/test_sweep.py b/scripts/tests/test_sweep.py index f66cd27..97a3d25 100644 --- a/scripts/tests/test_sweep.py +++ b/scripts/tests/test_sweep.py @@ -69,6 +69,10 @@ def gh(issues=(), refs=(), routes=None): return FakeGitHub(r) +def listed(n, age=timedelta(hours=1)): + return {"repo": "o/r", "issue": n, "submitted_at": iso(NOW - age)} + + def commit(sha, age): return {f"GET {API}/commits/{sha}": {"commit": {"committer": {"date": iso(NOW - age)}}}} @@ -94,6 +98,47 @@ def deleted(g): class SweepTest(unittest.TestCase): + def test_listed_submission_is_finished_not_redispatched(self): + # 9's branch is merged and won't delete; 8 already got its label; 7 merged too recently to touch. + def boom(b): + raise GitHubError(422, "nope") + g = gh([issue(9, age=timedelta(hours=2)), issue(8, labels=("submission", "accepted")), + issue(7, age=timedelta(hours=3))], + refs=[ref(9)], routes={**commit("sha9", timedelta(hours=2)), + f"DELETE {API}/git/refs/heads/submit/9": boom, + f"POST {API}/issues/9/comments": {}, f"POST {API}/issues/9/labels": {}, + f"PATCH {API}/issues/9": {}, f"PATCH {API}/issues/8": {}}) + sweep(g, NOW, [listed(9), listed(8), listed(7, age=timedelta(minutes=20))]) + writes = [(m, p) for m, p, _ in g.calls if m in ("POST", "PATCH")] + self.assertEqual(writes, [ + ("POST", f"{API}/issues/9/labels"), ("POST", f"{API}/issues/9/comments"), + ("PATCH", f"{API}/issues/9"), + ("PATCH", f"{API}/issues/8"), + ("POST", f"{API}/actions/workflows/submit.yml/dispatches"), + ]) + body = {p: b for m, p, b in g.calls} + self.assertIn("`o/r` is on the list", body[f"{API}/issues/9/comments"]["body"]) + self.assertEqual(body[f"{API}/issues/9"], {"state": "closed", "state_reason": "completed"}) + self.assertEqual(dispatched(g), ["7"]) + + def test_failed_comment_is_not_sent_again(self): + def boom(b): + raise GitHubError(502, "timeout") + g = gh([issue(9)], routes={f"POST {API}/issues/9/labels": {}, f"POST {API}/issues/9/comments": boom}) + sweep(g, NOW, [listed(9)]) + # Next sweep the label is on, so only the close is tried. + g = gh([issue(9, labels=("submission", "accepted"))], routes={f"PATCH {API}/issues/9": {}}) + sweep(g, NOW, [listed(9)]) + self.assertEqual([(m, p) for m, p, _ in g.calls if m in ("POST", "PATCH")], [("PATCH", f"{API}/issues/9")]) + + def test_failed_close_is_logged_and_the_sweep_goes_on(self): + def boom(b): + raise GitHubError(500, "boom") + g = gh([issue(9, labels=("submission", "accepted")), issue(5, age=timedelta(hours=3))], + routes={f"PATCH {API}/issues/9": boom}) + sweep(g, NOW, [listed(9)]) + self.assertEqual(dispatched(g), ["5"]) + def test_skips_handled_young_and_pr_issues(self): g = gh([ issue(2, labels=("submission", "accepted")),