Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions .github/workflows/submit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -132,14 +132,14 @@ jobs:
print(f"repo={repo}")
print(f"issue={issue}")
PY
# Prints the branch name, or nothing when fresh main already has the repo
# (result.json is then rewritten as a rejection).
# Prints the branch name, "done" when another run of this issue already added it,
# or nothing when fresh main already has the repo (result.json is then rewritten as a rejection).
- id: apply
env:
RESULT: ${{ runner.temp }}/result/result.json
run: |
branch=$(python -m scripts.submit apply --result "$RESULT")
echo "branch=$branch" >> "$GITHUB_OUTPUT"
out=$(python -m scripts.submit apply --result "$RESULT")
if [ "$out" = done ]; then echo "done=true"; else echo "branch=$out"; fi >> "$GITHUB_OUTPUT"
- id: pr
if: steps.apply.outputs.branch != ''
env:
Expand Down Expand Up @@ -224,7 +224,7 @@ jobs:
env:
GITHUB_TOKEN: ${{ steps.discussions_token.outputs.token }}
run: python -m scripts.on_merge
- if: steps.apply.outputs.branch == ''
- if: steps.apply.outputs.branch == '' && steps.apply.outputs.done != 'true'
env:
GH_TOKEN: ${{ steps.late_issues_token.outputs.token || steps.issues_token.outputs.token }}
RESULT: ${{ runner.temp }}/result/result.json
Expand Down
11 changes: 11 additions & 0 deletions .github/workflows/sweep.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@ on:
schedule:
- cron: "*/15 * * * *"
workflow_dispatch:
# A finished submission or report frees list-writes, so the next one is dispatched now,
# not at the next 15-minute slot. Neither workflow runs on pull requests, and this job
# checks out main and never reads the triggering run's head.
workflow_run:
workflows: [Submit, Reports]
types: [completed]

permissions: {}

Expand All @@ -17,6 +23,11 @@ concurrency:

jobs:
sweep:
# Only after a success: a failed or rate-limited run would otherwise re-dispatch in a tight loop.
if: >-
github.event_name != 'workflow_run' || (
github.event.workflow_run.conclusion == 'success' &&
github.event.workflow_run.head_branch == 'main')
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
Expand Down
5 changes: 4 additions & 1 deletion scripts/devstream/discussions.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@
BOT_LOGIN = "devstreamhq-bot"
SITE = "https://devstream.jamesconway.dev"
REMOVED_PREFIX = "Removed: "
# The thread's trust line, matching trust.py and the site. COUNTS_OLD is what threads made before it say.
COUNTS = "Each account counts once, if it's at least 30 days old and has a public repo or a contribution in the last year."
COUNTS_OLD = "Every GitHub account counts once."

_CATEGORY = """query LikesCategory {
repository(owner: "devstreamhq", name: "devstream") {
Expand Down Expand Up @@ -69,7 +72,7 @@ def ensure_discussion(entry, gh, repo_id: str, category_id: str, existing: dict[
body = (
f"{entry['description'].replace('@', '@' + chr(0x200b))}\n\n"
f"**Repo:** https://github.com/{repo}\n\n"
"React with 👍 to Like it. Every GitHub account counts once.\n\n"
f"React with 👍 to Like it. {COUNTS}\n\n"
f"See where it ranks: {SITE}"
)
data = gh.graphql(
Expand Down
28 changes: 27 additions & 1 deletion scripts/devstream/quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
)
# Title lines like "foo - DEPRECATED" or "# foo (deprecated)"; first non-blank line only.
README_TITLE_DEPRECATED = re.compile(r"[-\u2013\u2014:|(\[][ \t]{0,3}(?P<d>" + _MARK + r")" + _END, re.I)
# GitHub's whole takedown message: an IP allow list or a token problem is also a 403, and a false match removes a live repo.
BLOCKED = "repository access blocked"
README_MIN = 300
README_SCAN = 200_000

Expand Down Expand Up @@ -168,9 +170,33 @@ def _old_enough(created_at, now) -> bool:
return False


def blocked(e: GitHubError) -> bool:
"""A repo GitHub itself took down (DMCA 451, or a terms-of-service block): gone for good, like a 404."""
if e.status == 451:
return True
if e.status != 403:
return False
try:
body = json.loads(e.message)
except ValueError:
body = {"message": e.message}
if not isinstance(body, dict):
return False
return isinstance(body.get("block"), dict) or str(body.get("message") or "").strip().lower() == BLOCKED


def get_repo(gh, path: str) -> dict | None:
try:
return gh.get(path)
except GitHubError as e:
if blocked(e):
return None
raise


def check_repo(full: str, gh, now: datetime, mode: Literal["submit", "rot"],
allowed_extra: frozenset = frozenset()) -> RepoResult:
repo = gh.get(f"/repos/{full}")
repo = get_repo(gh, f"/repos/{full}")
if repo is None:
return RepoResult(False, full, [MSG_GONE], "remove")
# A redirect's final path is often /repositories/{id}, so the name comes from the body.
Expand Down
9 changes: 5 additions & 4 deletions scripts/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,9 @@
"so it's waiting for a maintainer to read instead."
)
RATE_LIMIT = "You've hit the daily limit of 5 reports. Try again tomorrow."
COUNTED = ("spam", "other")
# A spam or other report counts toward a hide for this long.
# Only spam counts toward an automatic hide; "Something else" goes to a maintainer like the rest.
COUNTED = ("spam",)
# A spam report counts toward a hide for this long.
REPORT_MAX_AGE = timedelta(days=30)
# hidden.json saying hidden while the entry is visible means "just un-hidden" only while on_merge is keeping it fresh.
JUST_UNHIDDEN = timedelta(hours=2)
Expand Down Expand Up @@ -150,7 +151,7 @@ def _counts_since(repo: str, gh, now: datetime) -> datetime:


def _reporters(repo: str, open_reports: list[dict], gh, since: datetime) -> set[str]:
"""Distinct human logins with an open, unedited spam or other report on repo, filed after since."""
"""Distinct human logins with an open, unedited spam report on repo, filed after since."""
found = {}
for r in open_reports:
if _is_bot(r) or "pull_request" in r or _parse(r["created_at"]) < since:
Expand Down Expand Up @@ -214,7 +215,7 @@ def handle_report(issue: dict, gh, entries: list, open_reports: list[dict],
if _established(login, gh, now):
return [Action("hide", repo)] + _reply(hidden, "needs-review")
return _reply(queued, "needs-review")
if reason in ("spam", "other"):
if reason in COUNTED:
counted = 0
others = [r for r in open_reports if r.get("number") != issue["number"]]
for who in sorted(_reporters(repo, [issue, *others], gh, _counts_since(repo, gh, now))):
Expand Down
12 changes: 9 additions & 3 deletions scripts/rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
MSG_SHORT_README,
Undecided,
check_repo,
get_repo,
)
from scripts.devstream.stats import ensure_branch, read_json, update_json
from scripts.validate_data import validate
Expand Down Expand Up @@ -228,9 +229,14 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict:
if e.get("repo_id") is None:
logging.warning("%s has no repo_id, so a replaced repo can't be caught", e["repo"])
elif r.exists and r.repo_id is not None and r.repo_id != e["repo_id"]:
# Someone else's repo at the listed name inherits nothing: it reads as gone, and a person decides.
replaced.append({"repo": e["repo"], "kind": "replaced", "want": e["repo_id"], "got": r.repo_id})
r = RepoResult(False, e["repo"], [MSG_REPLACED], "remove")
got = r.repo_id
# The owner may have moved (an account rename) and someone took the old name: follow the ID.
moved = get_repo(gh, f"/repositories/{e['repo_id']}")
r = moved and check(moved["full_name"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"]))
if not (r and r.exists and r.repo_id == e["repo_id"]):
# Someone else's repo at the listed name inherits nothing: it reads as gone, and a person decides.
replaced.append({"repo": e["repo"], "kind": "replaced", "want": e["repo_id"], "got": got})
r = RepoResult(False, e["repo"], [MSG_REPLACED], "remove")
results[e["repo"]] = r
except GitHubError as exc:
# An owner's IP allow list hides the repo from our token; that says nothing about the repo.
Expand Down
22 changes: 17 additions & 5 deletions scripts/submit.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""Decide on a repo submission issue, and apply an accepted one to entries.json.

python -m scripts.submit decide (--event EVENT.json | --issue N) --out result.json
python -m scripts.submit apply --result result.json
python -m scripts.submit apply --result result.json (prints the branch, "done" if this issue is already listed,
or nothing when it went down the reject path)
python -m scripts.submit decide ... --undecided undecided.json (exits 3 when the run reached no decision)
python -m scripts.submit undecided --file undecided.json (counts it, and parks the issue after 3)
python -m scripts.submit comment --result result.json --kind {accepted,rejected,pr,error}
Expand Down Expand Up @@ -57,10 +58,12 @@ def _dup_reason(e: Entry) -> str:
return ARCHIVED_DUP if e["status"] in ("archived", "superseded") else DUPLICATE


def _recent_submissions(login: str, gh, now: datetime) -> int:
def _other_recent_submissions(login: str, number: int, gh, now: datetime) -> int:
"""Excludes the current issue, so search lag can't let an extra one through."""
since = (now - timedelta(days=7)).strftime("%Y-%m-%d")
q = f"repo:{REPO}+label:submission+author:{login}+created:>={since}"
return gh.get(f"/search/issues?q={q}")["total_count"]
# shortcut: one page of 100 is plenty to see 3.
return sum(1 for i in gh.get(f"/search/issues?q={q}&per_page=100")["items"] if i["number"] != number)


def decide(issue: dict, gh, now: datetime, entries: list[Entry], excluded: set[str]) -> Decision:
Expand All @@ -77,7 +80,7 @@ def decide(issue: dict, gh, now: datetime, entries: list[Entry], excluded: set[s

reasons = []
# check_user has already rejected logins outside [A-Za-z0-9-], so the query is safe.
if _recent_submissions(login, gh, now) > WEEKLY_LIMIT:
if _other_recent_submissions(login, issue["number"], gh, now) >= WEEKLY_LIMIT:
reasons.append(RATE_LIMIT)

more, canonical, description, repo_id = _check(form["repository"], form["description"], gh, now, entries,
Expand Down Expand Up @@ -203,8 +206,13 @@ def _decide_issue(args) -> int:
else:
with open(args.event, encoding="utf-8") as f:
issue = json.load(f)["issue"]
entries = load()
# A second dispatch of a submission another run already added: deciding again would reject it.
if any(e["issue"] == issue["number"] for e in entries):
print(f"issue {issue['number']} is already on the list", file=sys.stderr)
return 0
excluded = {u.strip() for u in os.environ.get("EXCLUDED_USERS", "").split(",") if u.strip()}
d = decide(issue, gh, datetime.now(timezone.utc), load(), excluded)
d = decide(issue, gh, datetime.now(timezone.utc), entries, excluded)
out = asdict(d) | {"issue": issue["number"], "author": issue["user"]["login"]}
with open(args.out, "w", encoding="utf-8") as f:
json.dump(out, f, indent=2, ensure_ascii=False)
Expand All @@ -220,6 +228,10 @@ def _apply(args) -> int:
entries = load(args.entries)
# main may have moved since decide ran; a duplicate now goes down the reject path.
dup = find(entries, result["entry"]["repo"])
if dup and dup["issue"] == result["issue"]:
# Another run of this same issue added it, and that run answers the issue.
print("done")
return 0
if dup:
result.update(ok=False, reasons=[_dup_reason(dup)], entry=None)
with open(args.result, "w", encoding="utf-8") as f:
Expand Down
9 changes: 7 additions & 2 deletions scripts/sweep.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,12 @@ def _dispatch_if_stale(gh, wf, now) -> bool:


def _list_write_waiting(gh) -> bool:
# These share the list-writes group, which holds one waiting run.
# These share the list-writes group, which holds one waiting run. A running one counts too:
# its issue isn't labelled yet, so a dispatch now would decide it a second time.
return any(
gh.get(f"{API}/actions/workflows/{wf}/runs?status={s}")["total_count"] > 0
for wf in ("submit.yml", "rot.yml", "on-merge.yml", "reports.yml") for s in ("queued", "pending")
for wf in ("submit.yml", "rot.yml", "on-merge.yml", "reports.yml")
for s in ("queued", "pending", "in_progress")
)


Expand Down Expand Up @@ -165,6 +167,8 @@ def sweep(gh, now: datetime, entries=()) -> None:
todo = []
# Older than the longest run (merge wait included), so the run that added it has finished.
listed = {e["issue"]: e for e in entries if now - _parse(e["submitted_at"]) > BRANCH_AGE}
# Too recent to finish, but already on the list: submit would skip a dispatch, and set off another sweep.
adding = {e["issue"] for e in entries} - listed.keys()
# Submissions and reports share the one dispatch; every handled report is closed or needs-review.
for label, workflow, done in (("submission", "submit.yml", {"accepted", "rejected", "needs-review"}),
("report", "reports.yml", {"needs-review"})):
Expand All @@ -177,6 +181,7 @@ def sweep(gh, now: datetime, entries=()) -> None:
continue
if (label not in labels or "pull_request" in issue or labels & done
or issue["number"] in busy or now - _parse(issue["created_at"]) <= ISSUE_AGE
or (label == "submission" and issue["number"] in adding)
# The bot's own Like-burst reports are for a maintainer, not reports.yml.
or (label == "report" and (user.get("login") == BOT or user.get("type") == "Bot"))):
continue
Expand Down
14 changes: 9 additions & 5 deletions scripts/sync_likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -279,17 +279,21 @@ def run(gh, entries, now: datetime, excluded: set[str]) -> None:

prev_meta = read_json(gh, "meta.json")[0]
# A repo GitHub hides from us this hour keeps its last card rather than losing stars and avatar.
listed = set(keys)
fetched = fetch_meta(gh, keys)
# Every shown entry gets one, so the Archive page has stars for archived and superseded entries too.
carded = {}
for e in entries:
if e["status"] != "hidden":
carded.setdefault(normalize_repo(e["repo"]), e)
fetched = fetch_meta(gh, sorted(carded))
replaced = set()
for k, m in list(fetched.items()):
got, want = m.pop("repo_id"), ranked[k].get("repo_id")
got, want = m.pop("repo_id"), carded[k].get("repo_id")
if want is not None and got != want:
# Someone else's repo at the listed name: its stars and avatar never reach the card or Trending.
logging.warning("%s now names repo %s, not the listed %s; keeping its last card", ranked[k]["repo"], got, want)
logging.warning("%s now names repo %s, not the listed %s; keeping its last card", carded[k]["repo"], got, want)
del fetched[k]
replaced.add(k)
meta = {k: v for k, v in (prev_meta or {}).items() if k in listed} | fetched
meta = {k: v for k, v in (prev_meta or {}).items() if k in carded} | fetched
reactors = fetch_reactors(gh, sorted({mapping[k] for k in keys if k in mapping}))
prev_flags = read_json(gh, "flags.json")[0]
flags = copy.deepcopy(prev_flags or {})
Expand Down
15 changes: 13 additions & 2 deletions scripts/tests/test_discussions.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import unittest

from scripts.devstream.discussions import ensure_discussion, likes_category, list_discussions
from scripts.devstream.discussions import COUNTS, COUNTS_OLD, ensure_discussion, likes_category, list_discussions
from scripts.devstream.gh import GitHubError
from scripts.tests.fake_gh import FakeGitHub

Expand All @@ -23,10 +23,21 @@ def test_creates_missing_only(self):
self.assertEqual(
v["body"],
"A tidy tool for tidy people\n\n**Repo:** https://github.com/octo/tool\n\n"
"React with 👍 to Like it. Every GitHub account counts once.\n\n"
"React with 👍 to Like it. Each account counts once, if it's at least 30 days old "
"and has a public repo or a contribution in the last year.\n\n"
"See where it ranks: https://devstream.jamesconway.dev",
)

def test_old_trust_line_is_exposed_for_a_backfill(self):
# Threads made before the wording fix say this; a one-off edit swaps it for COUNTS.
self.assertEqual(COUNTS_OLD, "Every GitHub account counts once.")
old_body = "d\n\n**Repo:** https://github.com/o/r\n\nReact with 👍 to Like it. Every GitHub account counts once.\n\n"
self.assertIn(COUNTS_OLD, old_body)
g = gh(7)
ensure_discussion(ENTRY, g, "R1", "C1", {})
self.assertIn(COUNTS, g.calls[0][2]["body"])
self.assertNotIn(COUNTS_OLD, g.calls[0][2]["body"])

def test_idempotent_when_existing(self):
g = gh()
self.assertEqual(ensure_discussion(ENTRY, g, "R1", "C1", {"octo/tool": 3}), 3)
Expand Down
15 changes: 15 additions & 0 deletions scripts/tests/test_likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -888,6 +888,21 @@ def test_replaced_repo_keeps_last_card_and_is_flagged(self):
self.assertTrue(likes["o/sq"]["flagged"], status)
self.assertFalse(likes["o/a"]["flagged"])

def test_archived_and_superseded_entries_keep_a_card(self):
# The Archive page shows their stars; only an entry gone from entries.json loses its card.
old = {"stars": 9, "language": "Go", "language_color": "#00ADD8",
"pushed_at": "2026-10-01T00:00:00Z", "avatar_url": "https://a/old.png"}
# A hidden entry is off every page while a maintainer reviews it, so it gets no card.
w = World([entry("o/a"), entry("o/arch", "archived"), entry("o/sup", "superseded"), entry("o/hid", "hidden")],
{"o/a": [node("x")]}, stars={"o/a": 3, "o/arch": 7, "o/sup": 4, "o/hid": 5},
stats={"meta.json": {"o/removed": old, "o/hid": old}})
w.run()
meta = w.written("meta.json")
self.assertEqual({k: v["stars"] for k, v in meta.items()}, {"o/a": 3, "o/arch": 7, "o/sup": 4})
[(_, _, variables)] = [c for c in w.gh.calls if c[1] == "RepoMeta"]
self.assertNotIn("hid", variables.values())
self.assertEqual(set(w.written("likes.json")), {"o/a"})

def test_meta_batched_per_50(self):
repos = {f"o/r{i:03}": [] for i in range(51)}
w = World([entry(r) for r in repos], repos)
Expand Down
26 changes: 26 additions & 0 deletions scripts/tests/test_quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,32 @@ def test_rot_gone_repo(self):
r = run(FakeGitHub({}), "rot")
self.assertEqual((r.exists, r.rot), (False, "remove"))

def test_dmca_or_tos_blocked_repo_reads_as_gone(self):
# Bodies GitHub sends for a DMCA takedown (451) and a terms-of-service block (403).
for code, body in ((451, '{"message":"Repository access blocked","block":{"reason":"dmca"}}'),
(403, '{"message":"Repository access blocked","block":{"reason":"tos"}}'),
(403, '{"message":"REPOSITORY ACCESS BLOCKED"}'),
(403, '{"message":"Forbidden","block":{"reason":"tos"}}')):
def blocked(_, code=code, body=body):
raise GitHubError(code, body)

for mode in ("submit", "rot"):
r = run(FakeGitHub({f"GET {R}": blocked}), mode)
self.assertEqual((r.exists, r.failures, r.rot), (False, ["This repo no longer exists."], "remove"))

def test_other_403_on_the_repo_still_raises(self):
# A false match would remove a live repo, so only GitHub's exact takedown phrase or block object counts.
for msg in ('{"message":"Resource not accessible by integration"}',
"the `alice` organization has an IP allow list enabled",
'{"message":"You have exceeded a secondary rate limit. Please wait a few minutes."}',
'{"message":"Access to this repository has been disabled by GitHub Staff."}',
'{"message":"Repository access blocked-ish"}', '{"message":"x","block":null}'):
def e403(_, msg=msg):
raise GitHubError(403, msg)

with self.assertRaises(GitHubError):
run(FakeGitHub({f"GET {R}": e403}), "rot")

def test_rot_skips_entry_rules(self):
g = gh({"fork": True, "parent": {"full_name": "b/o", "default_branch": "m"}, "description": ""},
topics=(), contributing=False, coc=False)
Expand Down
Loading
Loading