Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/workflows/submit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -194,15 +194,20 @@ jobs:
client-id: ${{ vars.APP_CLIENT_ID }}
private-key: ${{ secrets.APP_PRIVATE_KEY }}
permission-issues: write
# The entry is live now, so a failure here must not park it as needs-review; the sweeper
# finishes the label and close if the issue is still open. The label goes on first and is
# retried; the comment is a create, so it is sent once and never retried.
- if: steps.apply.outputs.branch != ''
continue-on-error: true
env:
GH_TOKEN: ${{ steps.late_issues_token.outputs.token || steps.issues_token.outputs.token }}
RESULT: ${{ runner.temp }}/result/result.json
ISSUE: ${{ steps.meta.outputs.issue }}
run: |
python -m scripts.submit comment --result "$RESULT" --kind accepted > "$RUNNER_TEMP/comment.md"
retry() { for i in 1 2 3; do "$@" && return; [ "$i" = 3 ] || sleep $((i * 10)); done; return 1; }
retry gh issue edit "$ISSUE" --add-label accepted
gh issue comment "$ISSUE" --body-file "$RUNNER_TEMP/comment.md"
gh issue edit "$ISSUE" --add-label accepted
# The new entry's Like thread, made while this run still holds list-writes so a burst of
# submissions can't starve it. A failure is left to the hourly on-merge run, not parked.
- id: discussions_token
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sweep.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
private-key: ${{ secrets.APP_PRIVATE_KEY }}
permission-actions: write
permission-contents: write
permission-issues: read
permission-issues: write
permission-pull-requests: write
- run: python -m scripts.sweep
env:
Expand Down
28 changes: 26 additions & 2 deletions scripts/sweep.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@
import re
from datetime import datetime, timedelta, timezone

from scripts.devstream.entries import load
from scripts.devstream.gh import GitHub, GitHubError
from scripts.submit import comment

OWNER = "devstreamhq"
API = f"/repos/{OWNER}/devstream"
Expand Down Expand Up @@ -113,7 +115,23 @@ def _list_write_waiting(gh) -> bool:
)


def sweep(gh, now: datetime) -> None:
def _finish_listed(gh, issue, entry) -> None:
"""The issue's entry is on the list, so its run merged; post what a failed follow-up step didn't, and close it."""
n = issue["number"]
try:
if "accepted" not in {lb["name"] for lb in issue["labels"]}:
# Label first: a comment is a create, sent at most once, and the label marks it as tried.
gh.post(f"{API}/issues/{n}/labels", {"labels": ["accepted"]})
gh.post(f"{API}/issues/{n}/comments", {"body": comment({"entry": entry}, "accepted")})
gh.patch(f"{API}/issues/{n}", {"state": "closed", "state_reason": "completed"})
except GitHubError as e:
# Tried again next sweep; one bad issue mustn't stop the dispatches.
print(f"could not close #{n}: {e}")
return
print(f"closed #{n}: already on the list")


def sweep(gh, now: datetime, entries=()) -> None:
# Branches first, so an issue freed by a deleted branch can be dispatched this run.
busy = _sweep_branches(gh, now)
_sweep_rot_branches(gh, now)
Expand All @@ -128,12 +146,18 @@ def sweep(gh, now: datetime) -> None:
print("dispatched rot.yml")
return
todo = []
# Older than the longest run (merge wait included), so the run that added it has finished.
listed = {e["issue"]: e for e in entries if now - _parse(e["submitted_at"]) > BRANCH_AGE}
# Submissions and reports share the one dispatch; every handled report is closed or needs-review.
for label, workflow, done in (("submission", "submit.yml", {"accepted", "rejected", "needs-review"}),
("report", "reports.yml", {"needs-review"})):
for issue in gh.paginate(f"{API}/issues?state=open&labels={label}&per_page=100"):
labels = {lb["name"] for lb in issue["labels"]}
user = issue.get("user") or {}
# Before the busy check: a merged branch that won't delete must not keep its issue open.
if label == "submission" and label in labels and issue["number"] in listed and "pull_request" not in issue:
_finish_listed(gh, issue, listed[issue["number"]])
continue
if (label not in labels or "pull_request" in issue or labels & done
or issue["number"] in busy or now - _parse(issue["created_at"]) <= ISSUE_AGE
# The bot's own Like-burst reports are for a maintainer, not reports.yml.
Expand All @@ -149,4 +173,4 @@ def sweep(gh, now: datetime) -> None:


if __name__ == "__main__":
sweep(GitHub(os.environ["GITHUB_TOKEN"]), datetime.now(timezone.utc))
sweep(GitHub(os.environ["GITHUB_TOKEN"]), datetime.now(timezone.utc), load())
45 changes: 45 additions & 0 deletions scripts/tests/test_sweep.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ def gh(issues=(), refs=(), routes=None):
return FakeGitHub(r)


def listed(n, age=timedelta(hours=1)):
return {"repo": "o/r", "issue": n, "submitted_at": iso(NOW - age)}


def commit(sha, age):
return {f"GET {API}/commits/{sha}": {"commit": {"committer": {"date": iso(NOW - age)}}}}

Expand All @@ -94,6 +98,47 @@ def deleted(g):


class SweepTest(unittest.TestCase):
def test_listed_submission_is_finished_not_redispatched(self):
# 9's branch is merged and won't delete; 8 already got its label; 7 merged too recently to touch.
def boom(b):
raise GitHubError(422, "nope")
g = gh([issue(9, age=timedelta(hours=2)), issue(8, labels=("submission", "accepted")),
issue(7, age=timedelta(hours=3))],
refs=[ref(9)], routes={**commit("sha9", timedelta(hours=2)),
f"DELETE {API}/git/refs/heads/submit/9": boom,
f"POST {API}/issues/9/comments": {}, f"POST {API}/issues/9/labels": {},
f"PATCH {API}/issues/9": {}, f"PATCH {API}/issues/8": {}})
sweep(g, NOW, [listed(9), listed(8), listed(7, age=timedelta(minutes=20))])
writes = [(m, p) for m, p, _ in g.calls if m in ("POST", "PATCH")]
self.assertEqual(writes, [
("POST", f"{API}/issues/9/labels"), ("POST", f"{API}/issues/9/comments"),
("PATCH", f"{API}/issues/9"),
("PATCH", f"{API}/issues/8"),
("POST", f"{API}/actions/workflows/submit.yml/dispatches"),
])
body = {p: b for m, p, b in g.calls}
self.assertIn("`o/r` is on the list", body[f"{API}/issues/9/comments"]["body"])
self.assertEqual(body[f"{API}/issues/9"], {"state": "closed", "state_reason": "completed"})
self.assertEqual(dispatched(g), ["7"])

def test_failed_comment_is_not_sent_again(self):
def boom(b):
raise GitHubError(502, "timeout")
g = gh([issue(9)], routes={f"POST {API}/issues/9/labels": {}, f"POST {API}/issues/9/comments": boom})
sweep(g, NOW, [listed(9)])
# Next sweep the label is on, so only the close is tried.
g = gh([issue(9, labels=("submission", "accepted"))], routes={f"PATCH {API}/issues/9": {}})
sweep(g, NOW, [listed(9)])
self.assertEqual([(m, p) for m, p, _ in g.calls if m in ("POST", "PATCH")], [("PATCH", f"{API}/issues/9")])

def test_failed_close_is_logged_and_the_sweep_goes_on(self):
def boom(b):
raise GitHubError(500, "boom")
g = gh([issue(9, labels=("submission", "accepted")), issue(5, age=timedelta(hours=3))],
routes={f"PATCH {API}/issues/9": boom})
sweep(g, NOW, [listed(9)])
self.assertEqual(dispatched(g), ["5"])

def test_skips_handled_young_and_pr_issues(self):
g = gh([
issue(2, labels=("submission", "accepted")),
Expand Down
Loading