diff --git a/.github/workflows/submit.yml b/.github/workflows/submit.yml index 8a71019..37fa461 100644 --- a/.github/workflows/submit.yml +++ b/.github/workflows/submit.yml @@ -132,14 +132,14 @@ jobs: print(f"repo={repo}") print(f"issue={issue}") PY - # Prints the branch name, or nothing when fresh main already has the repo - # (result.json is then rewritten as a rejection). + # Prints the branch name, "done" when another run of this issue already added it, + # or nothing when fresh main already has the repo (result.json is then rewritten as a rejection). - id: apply env: RESULT: ${{ runner.temp }}/result/result.json run: | - branch=$(python -m scripts.submit apply --result "$RESULT") - echo "branch=$branch" >> "$GITHUB_OUTPUT" + out=$(python -m scripts.submit apply --result "$RESULT") + if [ "$out" = done ]; then echo "done=true"; else echo "branch=$out"; fi >> "$GITHUB_OUTPUT" - id: pr if: steps.apply.outputs.branch != '' env: @@ -224,7 +224,7 @@ jobs: env: GITHUB_TOKEN: ${{ steps.discussions_token.outputs.token }} run: python -m scripts.on_merge - - if: steps.apply.outputs.branch == '' + - if: steps.apply.outputs.branch == '' && steps.apply.outputs.done != 'true' env: GH_TOKEN: ${{ steps.late_issues_token.outputs.token || steps.issues_token.outputs.token }} RESULT: ${{ runner.temp }}/result/result.json diff --git a/.github/workflows/sweep.yml b/.github/workflows/sweep.yml index 82f787c..ae8e5c9 100644 --- a/.github/workflows/sweep.yml +++ b/.github/workflows/sweep.yml @@ -8,6 +8,12 @@ on: schedule: - cron: "*/15 * * * *" workflow_dispatch: + # A finished submission or report frees list-writes, so the next one is dispatched now, + # not at the next 15-minute slot. Neither workflow runs on pull requests, and this job + # checks out main and never reads the triggering run's head. + workflow_run: + workflows: [Submit, Reports] + types: [completed] permissions: {} @@ -17,6 +23,11 @@ concurrency: jobs: sweep: + # Only after a success: a failed or rate-limited run would otherwise re-dispatch in a tight loop. + if: >- + github.event_name != 'workflow_run' || ( + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_branch == 'main') runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/scripts/devstream/discussions.py b/scripts/devstream/discussions.py index 2d3210f..ad850bd 100644 --- a/scripts/devstream/discussions.py +++ b/scripts/devstream/discussions.py @@ -4,6 +4,9 @@ BOT_LOGIN = "devstreamhq-bot" SITE = "https://devstream.jamesconway.dev" REMOVED_PREFIX = "Removed: " +# The thread's trust line, matching trust.py and the site. COUNTS_OLD is what threads made before it say. +COUNTS = "Each account counts once, if it's at least 30 days old and has a public repo or a contribution in the last year." +COUNTS_OLD = "Every GitHub account counts once." _CATEGORY = """query LikesCategory { repository(owner: "devstreamhq", name: "devstream") { @@ -69,7 +72,7 @@ def ensure_discussion(entry, gh, repo_id: str, category_id: str, existing: dict[ body = ( f"{entry['description'].replace('@', '@' + chr(0x200b))}\n\n" f"**Repo:** https://github.com/{repo}\n\n" - "React with 👍 to Like it. Every GitHub account counts once.\n\n" + f"React with 👍 to Like it. {COUNTS}\n\n" f"See where it ranks: {SITE}" ) data = gh.graphql( diff --git a/scripts/devstream/quality.py b/scripts/devstream/quality.py index 4423f72..821dfd8 100644 --- a/scripts/devstream/quality.py +++ b/scripts/devstream/quality.py @@ -25,6 +25,8 @@ ) # Title lines like "foo - DEPRECATED" or "# foo (deprecated)"; first non-blank line only. README_TITLE_DEPRECATED = re.compile(r"[-\u2013\u2014:|(\[][ \t]{0,3}(?P" + _MARK + r")" + _END, re.I) +# GitHub's whole takedown message: an IP allow list or a token problem is also a 403, and a false match removes a live repo. +BLOCKED = "repository access blocked" README_MIN = 300 README_SCAN = 200_000 @@ -168,9 +170,33 @@ def _old_enough(created_at, now) -> bool: return False +def blocked(e: GitHubError) -> bool: + """A repo GitHub itself took down (DMCA 451, or a terms-of-service block): gone for good, like a 404.""" + if e.status == 451: + return True + if e.status != 403: + return False + try: + body = json.loads(e.message) + except ValueError: + body = {"message": e.message} + if not isinstance(body, dict): + return False + return isinstance(body.get("block"), dict) or str(body.get("message") or "").strip().lower() == BLOCKED + + +def get_repo(gh, path: str) -> dict | None: + try: + return gh.get(path) + except GitHubError as e: + if blocked(e): + return None + raise + + def check_repo(full: str, gh, now: datetime, mode: Literal["submit", "rot"], allowed_extra: frozenset = frozenset()) -> RepoResult: - repo = gh.get(f"/repos/{full}") + repo = get_repo(gh, f"/repos/{full}") if repo is None: return RepoResult(False, full, [MSG_GONE], "remove") # A redirect's final path is often /repositories/{id}, so the name comes from the body. diff --git a/scripts/reports.py b/scripts/reports.py index b997b65..6fcce5b 100644 --- a/scripts/reports.py +++ b/scripts/reports.py @@ -48,8 +48,9 @@ "so it's waiting for a maintainer to read instead." ) RATE_LIMIT = "You've hit the daily limit of 5 reports. Try again tomorrow." -COUNTED = ("spam", "other") -# A spam or other report counts toward a hide for this long. +# Only spam counts toward an automatic hide; "Something else" goes to a maintainer like the rest. +COUNTED = ("spam",) +# A spam report counts toward a hide for this long. REPORT_MAX_AGE = timedelta(days=30) # hidden.json saying hidden while the entry is visible means "just un-hidden" only while on_merge is keeping it fresh. JUST_UNHIDDEN = timedelta(hours=2) @@ -150,7 +151,7 @@ def _counts_since(repo: str, gh, now: datetime) -> datetime: def _reporters(repo: str, open_reports: list[dict], gh, since: datetime) -> set[str]: - """Distinct human logins with an open, unedited spam or other report on repo, filed after since.""" + """Distinct human logins with an open, unedited spam report on repo, filed after since.""" found = {} for r in open_reports: if _is_bot(r) or "pull_request" in r or _parse(r["created_at"]) < since: @@ -214,7 +215,7 @@ def handle_report(issue: dict, gh, entries: list, open_reports: list[dict], if _established(login, gh, now): return [Action("hide", repo)] + _reply(hidden, "needs-review") return _reply(queued, "needs-review") - if reason in ("spam", "other"): + if reason in COUNTED: counted = 0 others = [r for r in open_reports if r.get("number") != issue["number"]] for who in sorted(_reporters(repo, [issue, *others], gh, _counts_since(repo, gh, now))): diff --git a/scripts/rot.py b/scripts/rot.py index cd423c2..e072360 100644 --- a/scripts/rot.py +++ b/scripts/rot.py @@ -27,6 +27,7 @@ MSG_SHORT_README, Undecided, check_repo, + get_repo, ) from scripts.devstream.stats import ensure_branch, read_json, update_json from scripts.validate_data import validate @@ -228,9 +229,14 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict: if e.get("repo_id") is None: logging.warning("%s has no repo_id, so a replaced repo can't be caught", e["repo"]) elif r.exists and r.repo_id is not None and r.repo_id != e["repo_id"]: - # Someone else's repo at the listed name inherits nothing: it reads as gone, and a person decides. - replaced.append({"repo": e["repo"], "kind": "replaced", "want": e["repo_id"], "got": r.repo_id}) - r = RepoResult(False, e["repo"], [MSG_REPLACED], "remove") + got = r.repo_id + # The owner may have moved (an account rename) and someone took the old name: follow the ID. + moved = get_repo(gh, f"/repositories/{e['repo_id']}") + r = moved and check(moved["full_name"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"])) + if not (r and r.exists and r.repo_id == e["repo_id"]): + # Someone else's repo at the listed name inherits nothing: it reads as gone, and a person decides. + replaced.append({"repo": e["repo"], "kind": "replaced", "want": e["repo_id"], "got": got}) + r = RepoResult(False, e["repo"], [MSG_REPLACED], "remove") results[e["repo"]] = r except GitHubError as exc: # An owner's IP allow list hides the repo from our token; that says nothing about the repo. diff --git a/scripts/submit.py b/scripts/submit.py index b2ac4c6..e605e64 100644 --- a/scripts/submit.py +++ b/scripts/submit.py @@ -1,7 +1,8 @@ """Decide on a repo submission issue, and apply an accepted one to entries.json. python -m scripts.submit decide (--event EVENT.json | --issue N) --out result.json -python -m scripts.submit apply --result result.json +python -m scripts.submit apply --result result.json (prints the branch, "done" if this issue is already listed, + or nothing when it went down the reject path) python -m scripts.submit decide ... --undecided undecided.json (exits 3 when the run reached no decision) python -m scripts.submit undecided --file undecided.json (counts it, and parks the issue after 3) python -m scripts.submit comment --result result.json --kind {accepted,rejected,pr,error} @@ -57,10 +58,12 @@ def _dup_reason(e: Entry) -> str: return ARCHIVED_DUP if e["status"] in ("archived", "superseded") else DUPLICATE -def _recent_submissions(login: str, gh, now: datetime) -> int: +def _other_recent_submissions(login: str, number: int, gh, now: datetime) -> int: + """Excludes the current issue, so search lag can't let an extra one through.""" since = (now - timedelta(days=7)).strftime("%Y-%m-%d") q = f"repo:{REPO}+label:submission+author:{login}+created:>={since}" - return gh.get(f"/search/issues?q={q}")["total_count"] + # shortcut: one page of 100 is plenty to see 3. + return sum(1 for i in gh.get(f"/search/issues?q={q}&per_page=100")["items"] if i["number"] != number) def decide(issue: dict, gh, now: datetime, entries: list[Entry], excluded: set[str]) -> Decision: @@ -77,7 +80,7 @@ def decide(issue: dict, gh, now: datetime, entries: list[Entry], excluded: set[s reasons = [] # check_user has already rejected logins outside [A-Za-z0-9-], so the query is safe. - if _recent_submissions(login, gh, now) > WEEKLY_LIMIT: + if _other_recent_submissions(login, issue["number"], gh, now) >= WEEKLY_LIMIT: reasons.append(RATE_LIMIT) more, canonical, description, repo_id = _check(form["repository"], form["description"], gh, now, entries, @@ -203,8 +206,13 @@ def _decide_issue(args) -> int: else: with open(args.event, encoding="utf-8") as f: issue = json.load(f)["issue"] + entries = load() + # A second dispatch of a submission another run already added: deciding again would reject it. + if any(e["issue"] == issue["number"] for e in entries): + print(f"issue {issue['number']} is already on the list", file=sys.stderr) + return 0 excluded = {u.strip() for u in os.environ.get("EXCLUDED_USERS", "").split(",") if u.strip()} - d = decide(issue, gh, datetime.now(timezone.utc), load(), excluded) + d = decide(issue, gh, datetime.now(timezone.utc), entries, excluded) out = asdict(d) | {"issue": issue["number"], "author": issue["user"]["login"]} with open(args.out, "w", encoding="utf-8") as f: json.dump(out, f, indent=2, ensure_ascii=False) @@ -220,6 +228,10 @@ def _apply(args) -> int: entries = load(args.entries) # main may have moved since decide ran; a duplicate now goes down the reject path. dup = find(entries, result["entry"]["repo"]) + if dup and dup["issue"] == result["issue"]: + # Another run of this same issue added it, and that run answers the issue. + print("done") + return 0 if dup: result.update(ok=False, reasons=[_dup_reason(dup)], entry=None) with open(args.result, "w", encoding="utf-8") as f: diff --git a/scripts/sweep.py b/scripts/sweep.py index 598f03d..8da038d 100644 --- a/scripts/sweep.py +++ b/scripts/sweep.py @@ -123,10 +123,12 @@ def _dispatch_if_stale(gh, wf, now) -> bool: def _list_write_waiting(gh) -> bool: - # These share the list-writes group, which holds one waiting run. + # These share the list-writes group, which holds one waiting run. A running one counts too: + # its issue isn't labelled yet, so a dispatch now would decide it a second time. return any( gh.get(f"{API}/actions/workflows/{wf}/runs?status={s}")["total_count"] > 0 - for wf in ("submit.yml", "rot.yml", "on-merge.yml", "reports.yml") for s in ("queued", "pending") + for wf in ("submit.yml", "rot.yml", "on-merge.yml", "reports.yml") + for s in ("queued", "pending", "in_progress") ) @@ -165,6 +167,8 @@ def sweep(gh, now: datetime, entries=()) -> None: todo = [] # Older than the longest run (merge wait included), so the run that added it has finished. listed = {e["issue"]: e for e in entries if now - _parse(e["submitted_at"]) > BRANCH_AGE} + # Too recent to finish, but already on the list: submit would skip a dispatch, and set off another sweep. + adding = {e["issue"] for e in entries} - listed.keys() # Submissions and reports share the one dispatch; every handled report is closed or needs-review. for label, workflow, done in (("submission", "submit.yml", {"accepted", "rejected", "needs-review"}), ("report", "reports.yml", {"needs-review"})): @@ -177,6 +181,7 @@ def sweep(gh, now: datetime, entries=()) -> None: continue if (label not in labels or "pull_request" in issue or labels & done or issue["number"] in busy or now - _parse(issue["created_at"]) <= ISSUE_AGE + or (label == "submission" and issue["number"] in adding) # The bot's own Like-burst reports are for a maintainer, not reports.yml. or (label == "report" and (user.get("login") == BOT or user.get("type") == "Bot"))): continue diff --git a/scripts/sync_likes.py b/scripts/sync_likes.py index 441a96a..b91d0f2 100644 --- a/scripts/sync_likes.py +++ b/scripts/sync_likes.py @@ -279,17 +279,21 @@ def run(gh, entries, now: datetime, excluded: set[str]) -> None: prev_meta = read_json(gh, "meta.json")[0] # A repo GitHub hides from us this hour keeps its last card rather than losing stars and avatar. - listed = set(keys) - fetched = fetch_meta(gh, keys) + # Every shown entry gets one, so the Archive page has stars for archived and superseded entries too. + carded = {} + for e in entries: + if e["status"] != "hidden": + carded.setdefault(normalize_repo(e["repo"]), e) + fetched = fetch_meta(gh, sorted(carded)) replaced = set() for k, m in list(fetched.items()): - got, want = m.pop("repo_id"), ranked[k].get("repo_id") + got, want = m.pop("repo_id"), carded[k].get("repo_id") if want is not None and got != want: # Someone else's repo at the listed name: its stars and avatar never reach the card or Trending. - logging.warning("%s now names repo %s, not the listed %s; keeping its last card", ranked[k]["repo"], got, want) + logging.warning("%s now names repo %s, not the listed %s; keeping its last card", carded[k]["repo"], got, want) del fetched[k] replaced.add(k) - meta = {k: v for k, v in (prev_meta or {}).items() if k in listed} | fetched + meta = {k: v for k, v in (prev_meta or {}).items() if k in carded} | fetched reactors = fetch_reactors(gh, sorted({mapping[k] for k in keys if k in mapping})) prev_flags = read_json(gh, "flags.json")[0] flags = copy.deepcopy(prev_flags or {}) diff --git a/scripts/tests/test_discussions.py b/scripts/tests/test_discussions.py index 85702ad..b3cc1fb 100644 --- a/scripts/tests/test_discussions.py +++ b/scripts/tests/test_discussions.py @@ -1,6 +1,6 @@ import unittest -from scripts.devstream.discussions import ensure_discussion, likes_category, list_discussions +from scripts.devstream.discussions import COUNTS, COUNTS_OLD, ensure_discussion, likes_category, list_discussions from scripts.devstream.gh import GitHubError from scripts.tests.fake_gh import FakeGitHub @@ -23,10 +23,21 @@ def test_creates_missing_only(self): self.assertEqual( v["body"], "A tidy tool for tidy people\n\n**Repo:** https://github.com/octo/tool\n\n" - "React with 👍 to Like it. Every GitHub account counts once.\n\n" + "React with 👍 to Like it. Each account counts once, if it's at least 30 days old " + "and has a public repo or a contribution in the last year.\n\n" "See where it ranks: https://devstream.jamesconway.dev", ) + def test_old_trust_line_is_exposed_for_a_backfill(self): + # Threads made before the wording fix say this; a one-off edit swaps it for COUNTS. + self.assertEqual(COUNTS_OLD, "Every GitHub account counts once.") + old_body = "d\n\n**Repo:** https://github.com/o/r\n\nReact with 👍 to Like it. Every GitHub account counts once.\n\n" + self.assertIn(COUNTS_OLD, old_body) + g = gh(7) + ensure_discussion(ENTRY, g, "R1", "C1", {}) + self.assertIn(COUNTS, g.calls[0][2]["body"]) + self.assertNotIn(COUNTS_OLD, g.calls[0][2]["body"]) + def test_idempotent_when_existing(self): g = gh() self.assertEqual(ensure_discussion(ENTRY, g, "R1", "C1", {"octo/tool": 3}), 3) diff --git a/scripts/tests/test_likes.py b/scripts/tests/test_likes.py index d4a0d8a..b2b9a28 100644 --- a/scripts/tests/test_likes.py +++ b/scripts/tests/test_likes.py @@ -888,6 +888,21 @@ def test_replaced_repo_keeps_last_card_and_is_flagged(self): self.assertTrue(likes["o/sq"]["flagged"], status) self.assertFalse(likes["o/a"]["flagged"]) + def test_archived_and_superseded_entries_keep_a_card(self): + # The Archive page shows their stars; only an entry gone from entries.json loses its card. + old = {"stars": 9, "language": "Go", "language_color": "#00ADD8", + "pushed_at": "2026-10-01T00:00:00Z", "avatar_url": "https://a/old.png"} + # A hidden entry is off every page while a maintainer reviews it, so it gets no card. + w = World([entry("o/a"), entry("o/arch", "archived"), entry("o/sup", "superseded"), entry("o/hid", "hidden")], + {"o/a": [node("x")]}, stars={"o/a": 3, "o/arch": 7, "o/sup": 4, "o/hid": 5}, + stats={"meta.json": {"o/removed": old, "o/hid": old}}) + w.run() + meta = w.written("meta.json") + self.assertEqual({k: v["stars"] for k, v in meta.items()}, {"o/a": 3, "o/arch": 7, "o/sup": 4}) + [(_, _, variables)] = [c for c in w.gh.calls if c[1] == "RepoMeta"] + self.assertNotIn("hid", variables.values()) + self.assertEqual(set(w.written("likes.json")), {"o/a"}) + def test_meta_batched_per_50(self): repos = {f"o/r{i:03}": [] for i in range(51)} w = World([entry(r) for r in repos], repos) diff --git a/scripts/tests/test_quality.py b/scripts/tests/test_quality.py index 0f925fb..b1d1dd2 100644 --- a/scripts/tests/test_quality.py +++ b/scripts/tests/test_quality.py @@ -431,6 +431,32 @@ def test_rot_gone_repo(self): r = run(FakeGitHub({}), "rot") self.assertEqual((r.exists, r.rot), (False, "remove")) + def test_dmca_or_tos_blocked_repo_reads_as_gone(self): + # Bodies GitHub sends for a DMCA takedown (451) and a terms-of-service block (403). + for code, body in ((451, '{"message":"Repository access blocked","block":{"reason":"dmca"}}'), + (403, '{"message":"Repository access blocked","block":{"reason":"tos"}}'), + (403, '{"message":"REPOSITORY ACCESS BLOCKED"}'), + (403, '{"message":"Forbidden","block":{"reason":"tos"}}')): + def blocked(_, code=code, body=body): + raise GitHubError(code, body) + + for mode in ("submit", "rot"): + r = run(FakeGitHub({f"GET {R}": blocked}), mode) + self.assertEqual((r.exists, r.failures, r.rot), (False, ["This repo no longer exists."], "remove")) + + def test_other_403_on_the_repo_still_raises(self): + # A false match would remove a live repo, so only GitHub's exact takedown phrase or block object counts. + for msg in ('{"message":"Resource not accessible by integration"}', + "the `alice` organization has an IP allow list enabled", + '{"message":"You have exceeded a secondary rate limit. Please wait a few minutes."}', + '{"message":"Access to this repository has been disabled by GitHub Staff."}', + '{"message":"Repository access blocked-ish"}', '{"message":"x","block":null}'): + def e403(_, msg=msg): + raise GitHubError(403, msg) + + with self.assertRaises(GitHubError): + run(FakeGitHub({f"GET {R}": e403}), "rot") + def test_rot_skips_entry_rules(self): g = gh({"fork": True, "parent": {"full_name": "b/o", "default_branch": "m"}, "description": ""}, topics=(), contributing=False, coc=False) diff --git a/scripts/tests/test_reports.py b/scripts/tests/test_reports.py index e4d214f..0f81169 100644 --- a/scripts/tests/test_reports.py +++ b/scripts/tests/test_reports.py @@ -167,10 +167,13 @@ def test_unhide_of_another_entry_does_not_matter(self): g = gh(extra=hidden_json({"alice/other": {"hidden": True, "unhidden_at": None, "seen_at": iso(NOW)}})) self.assertIn("hide", kinds(run(g=g, open_reports=[report(8, "dave"), report(9, "erin")]))) - def test_something_else_counts_the_same_way(self): + def test_something_else_never_hides_but_goes_to_review(self): others = [report(8, "dave"), report(9, "erin")] actions = run(report(reason="Something else"), open_reports=others) - self.assertIn("hide", kinds(actions)) + self.assertNotIn("hide", kinds(actions)) + self.assertEqual(labels(actions), {"needs-review"}) + self.assertIn("a maintainer will take a look", comments(actions)[0]) + self.assertNotIn("close", kinds(actions)) def test_same_reporter_thrice_does_not_hide(self): others = [report(8, "carol"), report(9, "Carol")] @@ -197,12 +200,12 @@ def test_trusted_but_not_established_does_not_count(self): self.assertNotIn("hide", kinds(actions)) self.assertEqual(labels(actions), {"needs-review"}) - def test_only_spam_and_other_reports_count(self): + def test_only_spam_reports_count(self): for reason in ("Wrong category", "Dead or abandoned", "Replaced by something better", - "Malicious (malware, phishing, stolen code)"): + "Malicious (malware, phishing, stolen code)", "Something else"): others = [report(8, "dave", reason=reason), report(9, "erin")] self.assertNotIn("hide", kinds(run(open_reports=others)), reason) - others = [report(8, "dave", reason="Something else"), report(9, "erin")] + others = [report(8, "dave", reason="Spam or SEO"), report(9, "erin")] self.assertIn("hide", kinds(run(open_reports=others))) def test_edited_report_does_not_count(self): diff --git a/scripts/tests/test_rot.py b/scripts/tests/test_rot.py index 709a68f..8d89f2b 100644 --- a/scripts/tests/test_rot.py +++ b/scripts/tests/test_rot.py @@ -318,6 +318,17 @@ def check(full, gh, now, mode, **kw): self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("warn", "bob/other")]) self.assertEqual({e["repo"]: e["status"] for e in load(self.path)}, {"alice/tool": "live", "bob/other": "warned"}) + def test_dmca_or_tos_blocked_repo_gets_the_gone_notice_not_an_abort(self): + for code in (451, 403): + def blocked(_, code=code): + raise GitHubError(code, '{"message":"Repository access blocked","block":{"reason":"dmca"}}') + + save([entry()], self.path) + plan = self.run_prepare(FakeGitHub(routes(extra={"GET /repos/alice/tool": blocked})), check_repo) + self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("notice", "alice/tool")], code) + self.assertIn("no longer exists", plan["actions"][0]["message"]) + self.assertEqual(plan["review"], []) + def test_other_403_still_aborts(self): def check(full, gh, now, mode, **kw): raise GitHubError(403, "Resource not accessible by integration") @@ -499,6 +510,40 @@ def check(full, gh, now, mode, **kw): self.assertIn("`alice/tool`", issues[0]["body"]) self.assertIn("999", issues[0]["body"]) + def test_owner_rename_with_a_squatter_at_the_old_name_follows_the_rename(self): + def check(full, gh, now, mode, **kw): + r = ok(full) + r.repo_id = {"alice/tool": 999, "alice2/tool": 1}.get(full, 2) + return r + + g = FakeGitHub(routes(extra={"GET /repositories/1": {"id": 1, "full_name": "alice2/tool"}})) + plan = self.run_prepare(g, check) + self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("rename", "alice2/tool")]) + self.assertEqual(plan["review"], []) + self.assertEqual(load(self.path)[0]["repo"], "alice2/tool") + + # finish moves the Likes onto the new name, as for any rename. + r, state = store({"likes.json": {"alice/tool": {"likes": 5}}, "discussions.json": {"alice/tool": 3}}, + extra=threads({3: "alice/tool"})) + finish(FakeGitHub(r), plan, NOW) + self.assertEqual(state["likes.json"], {"alice2/tool": {"likes": 5}}) + + def test_squatter_at_the_name_of_a_gone_or_blocked_repo_is_replaced(self): + def check(full, gh, now, mode, **kw): + r = ok(full) + r.repo_id = 999 if full == "alice/tool" else 2 + return r + + def blocked(_): + raise GitHubError(451, '{"message":"Repository access blocked"}') + + for gone_route in ({}, {"GET /repositories/1": blocked}): + save([entry(), entry("bob/other", issue=2)], self.path) + plan = self.run_prepare(FakeGitHub(routes(extra=gone_route)), check) + self.assertEqual(kinds([Action(**a) for a in plan["actions"]]), [("notice", "alice/tool")]) + self.assertIn("A different repo now", plan["actions"][0]["message"]) + self.assertEqual(plan["review"], [{"repo": "alice/tool", "kind": "replaced", "want": 1, "got": 999}]) + def test_missing_repo_id_warns_and_carries_on(self): e = entry() del e["repo_id"] diff --git a/scripts/tests/test_submit.py b/scripts/tests/test_submit.py index da3e750..d40607d 100644 --- a/scripts/tests/test_submit.py +++ b/scripts/tests/test_submit.py @@ -7,6 +7,7 @@ import time import unittest import zlib +from pathlib import Path from unittest import mock from datetime import datetime, timedelta, timezone @@ -27,7 +28,7 @@ R = "/repos/alice/tool" SEARCH = ( "GET /search/issues?q=repo:devstreamhq/devstream+label:submission" - "+author:carol+created:>=2026-10-29" + "+author:carol+created:>=2026-10-29&per_page=100" ) @@ -45,10 +46,11 @@ def issue(b=None, title="Submit: alice/tool"): def gh(count=1, repo_kw=None, extra=None): + """count: the author's submissions this week, the current issue (7) included.""" routes = { "GET /orgs/devstreamhq/blocks": [], "GET /users/carol": {"created_at": iso(NOW - timedelta(days=400)), "public_repos": 3, "type": "User"}, - SEARCH: {"total_count": count, "items": []}, + SEARCH: {"total_count": count, "items": [{"number": 7}] + [{"number": 100 + k} for k in range(count - 1)]}, f"GET {R}": repo(**(repo_kw or {})), f"GET {R}/community/profile": {"files": {"contributing": {}, "code_of_conduct": {}}}, f"GET {R}/readme": b64(README), @@ -113,6 +115,12 @@ def test_fourth_submission_in_week_rejected(self): self.assertEqual(d.reasons, [RATE_LIMIT]) self.assertEqual(RATE_LIMIT, "You've hit the limit of 3 submissions a week. Try again in a few days.") + def test_weekly_limit_does_not_rely_on_search_seeing_this_issue(self): + # Search lags: three earlier submissions, and this one not indexed yet. + lagging = {"total_count": 3, "items": [{"number": 100}, {"number": 101}, {"number": 102}]} + d = run(g=gh(extra={SEARCH: lagging})) + self.assertEqual(d.reasons, [RATE_LIMIT]) + def test_collects_multiple_repo_failures(self): g = gh(repo_kw={"archived": True, "license": None, "pushed_at": iso(NOW - timedelta(days=200))}) d = run(issue(body(description="short")), g) @@ -123,6 +131,14 @@ def test_collects_multiple_repo_failures(self): self.assertIn("This repo hasn't had a push in the last 90 days.", d.reasons) self.assertTrue(any("license" in r for r in d.reasons)) + def test_dmca_blocked_repo_is_rejected_not_parked(self): + def blocked(_): + raise GitHubError(451, '{"message":"Repository access blocked","block":{"reason":"dmca"}}') + + d = run(g=gh(extra={f"GET {R}": blocked})) + self.assertFalse(d.ok) + self.assertIn(NOT_FOUND, d.reasons) + def test_form_failure_stops_early(self): g = gh() d = run(issue("### Repository\n\nalice/tool\n"), g) @@ -289,6 +305,17 @@ def test_dispatch_skips_handled_or_foreign_issues(self): self.assertEqual(self.main(g, "--issue", "7"), 0) self.assertFalse(self.wrote) + def test_issue_already_on_the_list_is_skipped(self): + listed = [dict(entry(), issue=7)] + with mock.patch("scripts.submit.load", lambda: listed): + g = gh(extra={"GET /repos/devstreamhq/devstream/issues/7": issue_api()}) + with tempfile.TemporaryDirectory() as tmp, mock.patch("scripts.submit.GitHub", lambda token: g), \ + mock.patch.dict(os.environ, {"GITHUB_TOKEN": "t"}): + out = os.path.join(tmp, "result.json") + self.assertEqual(main(["decide", "--issue", "7", "--out", out]), 0) + self.assertFalse(os.path.exists(out)) + self.assertEqual([c[1] for c in g.calls], ["/repos/devstreamhq/devstream/issues/7"]) + API = "/repos/devstreamhq/devstream" BOT = "devstreamhq-bot[bot]" @@ -409,6 +436,28 @@ def test_apply_duplicate_on_fresh_main_routes_to_reject(self): self.assertEqual(json.load(f), {"ok": False, "reasons": [reason], "entry": None, "issue": 7, "author": "carol"}) + def test_apply_of_an_issue_already_on_main_is_done_not_rejected(self): + # A duplicate dispatch of the same submission: the first run already added it. + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "entries.json") + save([dict(entry("Alice/Tool"), issue=7)], path) + before = open(path).read() + result = os.path.join(tmp, "result.json") + accepted = {"ok": True, "reasons": [], "entry": run().entry, "issue": 7, "author": "carol"} + write(result, accepted) + p = subprocess.run([sys.executable, "-m", "scripts.submit", "apply", "--result", result, + "--entries", path], capture_output=True, text=True) + self.assertEqual((p.returncode, p.stdout.strip()), (0, "done"), p.stderr) + self.assertEqual(open(path).read(), before) + with open(result) as f: + self.assertEqual(json.load(f), accepted) + + def test_workflow_skips_the_reject_step_when_apply_is_done(self): + text = (Path(__file__).resolve().parents[2] / ".github" / "workflows" / "submit.yml").read_text() + apply = text[text.index("\n apply:\n"):text.index("\n reject:\n")] + step = apply[apply.rindex("- if:", 0, apply.index("--kind rejected")):] + self.assertIn("steps.apply.outputs.done != 'true'", step[:step.index("run:")]) + def test_apply_appends_and_validates(self): with tempfile.TemporaryDirectory() as tmp: path = os.path.join(tmp, "entries.json") diff --git a/scripts/tests/test_sweep.py b/scripts/tests/test_sweep.py index 56e95a0..e8e5022 100644 --- a/scripts/tests/test_sweep.py +++ b/scripts/tests/test_sweep.py @@ -1,5 +1,6 @@ import unittest from datetime import datetime, timedelta, timezone +from pathlib import Path from scripts.devstream.gh import GitHubError from scripts.sweep import sweep @@ -18,6 +19,9 @@ MERGE_PENDING = f"GET {API}/actions/workflows/on-merge.yml/runs?status=pending" REPORT_QUEUED = f"GET {API}/actions/workflows/reports.yml/runs?status=queued" REPORT_PENDING = f"GET {API}/actions/workflows/reports.yml/runs?status=pending" +# A run that has started still holds list-writes, and its issue isn't labelled yet. +RUNNING = {wf: f"GET {API}/actions/workflows/{wf}/runs?status=in_progress" + for wf in ("submit.yml", "rot.yml", "on-merge.yml", "reports.yml")} REPORT_DISPATCH = f"POST {API}/actions/workflows/reports.yml/dispatches" ROT_OK = f"GET {API}/actions/workflows/rot.yml/runs?status=success&per_page=100" ROT_LATEST = f"GET {API}/actions/workflows/rot.yml/runs?per_page=100" @@ -66,6 +70,7 @@ def gh(issues=(), refs=(), routes=None): ROT_QUEUED: {"total_count": 0}, ROT_PENDING: {"total_count": 0}, MERGE_QUEUED: {"total_count": 0}, MERGE_PENDING: {"total_count": 0}, REPORT_QUEUED: {"total_count": 0}, REPORT_PENDING: {"total_count": 0}, REPORT_DISPATCH: None, + **{k: {"total_count": 0} for k in RUNNING.values()}, ROT_OK: runs(timedelta(days=2)), ROT_LATEST: runs(timedelta(days=2)), ROT_DISPATCH: None, ROT_REFS: [], OPEN_PRS: [], LIKES_RUNS: runs(timedelta(minutes=20)), MERGE_RUNS: runs(timedelta(minutes=20)), @@ -109,7 +114,8 @@ def deleted(g): class SweepTest(unittest.TestCase): def test_listed_submission_is_finished_not_redispatched(self): - # 9's branch is merged and won't delete; 8 already got its label; 7 merged too recently to touch. + # 9's branch is merged and won't delete; 8 already got its label; 7 merged too recently to touch, + # and is never dispatched: deciding it again would reject a listed repo as a duplicate. def boom(b): raise GitHubError(422, "nope") g = gh([issue(9, age=timedelta(hours=2)), issue(8, labels=("submission", "accepted")), @@ -124,12 +130,11 @@ def boom(b): ("POST", f"{API}/issues/9/labels"), ("POST", f"{API}/issues/9/comments"), ("PATCH", f"{API}/issues/9"), ("PATCH", f"{API}/issues/8"), - ("POST", f"{API}/actions/workflows/submit.yml/dispatches"), ]) body = {p: b for m, p, b in g.calls} self.assertIn("`o/r` is on the list", body[f"{API}/issues/9/comments"]["body"]) self.assertEqual(body[f"{API}/issues/9"], {"state": "closed", "state_reason": "completed"}) - self.assertEqual(dispatched(g), ["7"]) + self.assertEqual(dispatched(g), []) def test_failed_comment_is_not_sent_again(self): def boom(b): @@ -214,6 +219,12 @@ def test_no_dispatch_while_a_submit_run_waits(self): sweep(g, NOW) self.assertEqual(dispatched(g), [], key) + def test_no_dispatch_while_a_list_writes_run_is_in_progress(self): + for key in RUNNING.values(): + g = gh([issue(1)], routes={key: {"total_count": 1}, ROT_OK: runs(), ROT_LATEST: runs()}) + sweep(g, NOW) + self.assertEqual((dispatched(g), rot_dispatched(g)), ([], []), key) + def test_no_dispatch_while_a_rot_run_waits(self): # Both share the list-writes group, so a new dispatch would cancel the waiting rot run. for key in (ROT_QUEUED, ROT_PENDING): @@ -398,3 +409,17 @@ def test_a_pending_submission_goes_before_a_stale_on_merge(self): g = gh([issue(9)], routes={MERGE_RUNS: runs(timedelta(hours=3))}) sweep(g, NOW) self.assertEqual((dispatched(g), hourly_dispatched(g)), (["9"], [])) + + +class WorkflowTest(unittest.TestCase): + def test_sweep_runs_when_a_submission_or_report_finishes(self): + wf = Path(__file__).resolve().parents[2] / ".github" / "workflows" + text = (wf / "sweep.yml").read_text() + on = text[text.index("workflow_run:"):text.index("types: [completed]")] + for name, file in (("Submit", "submit.yml"), ("Reports", "reports.yml")): + self.assertIn(name, on) + self.assertTrue((wf / file).read_text().startswith(f"name: {name}\n"), file) + # Never the triggering run's code, and never a loop on a failing run. + self.assertIn("github.event.workflow_run.conclusion == 'success'", text) + self.assertNotIn("workflow_run.head_sha", text) + self.assertIn("ref: main", text)