Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
454 changes: 454 additions & 0 deletions data/entries.json

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions data/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"additionalProperties": false,
"required": [
"repo",
"repo_id",
"category",
"description",
"submitted_by",
Expand All @@ -38,6 +39,11 @@
"type": "string",
"pattern": "^[A-Za-z0-9-]+/[A-Za-z0-9._-]+$"
},
"repo_id": {
"type": "integer",
"minimum": 1,
"description": "GitHub's numeric repo ID, which never changes or gets reused. The health check compares it to catch a different repo at the same name."
},
"category": {
"enum": [
"ai-ml",
Expand Down
1 change: 1 addition & 0 deletions scripts/devstream/entries.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

class Entry(TypedDict):
repo: str
repo_id: int
category: str
description: str
submitted_by: str
Expand Down
5 changes: 3 additions & 2 deletions scripts/devstream/gh.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,13 +151,14 @@ def status(self, path) -> int:
return 404
return 200 if r[1] is not None else 204

def paginate(self, path, params=None) -> Iterator[dict]:
def paginate(self, path, params=None, must_exist=False) -> Iterator[dict]:
"""must_exist: a 404 on the first page raises instead of reading as an empty list."""
self.last_redirect = None
first = True
while path:
r = self._request("GET", path, params)
if r is None:
if first:
if first and not must_exist:
return
# A vanished page would silently truncate the results.
raise GitHubError(404, f"page not found: {path}")
Expand Down
3 changes: 2 additions & 1 deletion scripts/devstream/likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"reactions(content: THUMBS_UP, first: 100{after}) "
"{{ nodes {{ createdAt user {{ login createdAt }} }} pageInfo {{ hasNextPage endCursor }} }}"
)
_META = "nameWithOwner stargazerCount primaryLanguage { name color } pushedAt owner { avatarUrl }"
_META = "nameWithOwner databaseId stargazerCount primaryLanguage { name color } pushedAt owner { avatarUrl }"


def count_likes(reactors: list[str], trusted: Callable[[str], bool]) -> int:
Expand Down Expand Up @@ -104,5 +104,6 @@ def fetch_meta(gh, repos: list[str]) -> dict[str, dict]:
"language_color": lang.get("color"),
"pushed_at": m["pushedAt"],
"avatar_url": m["owner"]["avatarUrl"],
"repo_id": m["databaseId"],
}
return out
5 changes: 3 additions & 2 deletions scripts/devstream/quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ class RepoResult:
failures: list[str] = field(default_factory=list)
rot: Literal["ok", "warn", "remove"] = "ok"
meta: dict = field(default_factory=dict)
repo_id: int | None = None


def _readme_text_length(text: str) -> int:
Expand Down Expand Up @@ -197,7 +198,7 @@ def check_repo(full: str, gh, now: datetime, mode: Literal["submit", "rot"],
rot = "warn"
else:
rot = "ok"
return RepoResult(True, r, failures, rot, meta)
return RepoResult(True, r, failures, rot, meta, repo.get("id"))

if age is None or age > timedelta(days=90):
failures.append(MSG_STALE)
Expand All @@ -214,4 +215,4 @@ def check_repo(full: str, gh, now: datetime, mode: Literal["submit", "rot"],
missing = _extras_missing(repo, gh, r)
if missing:
failures.append("Add at least two of these to the repo: " + ", ".join(missing) + ".")
return RepoResult(True, r, failures, "ok", meta)
return RepoResult(True, r, failures, "ok", meta, repo.get("id"))
25 changes: 10 additions & 15 deletions scripts/devstream/trust.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import tempfile
from datetime import datetime, timedelta

from scripts.devstream.gh import GitHubError

ORG = "devstreamhq"
LOGIN_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$")
Expand Down Expand Up @@ -37,15 +36,16 @@ def _parse(s: str) -> datetime:


def _check(login, gh, now, excluded):
"""Return (reason or None, created_at or None). API errors propagate."""
"""Return (reason or None, created_at or None). API errors propagate.

excluded must already hold the org's blocked logins (blocked_logins), which read as BLOCKED.
"""
if login.lower() in {e.lower() for e in excluded}:
return BLOCKED, None
if login.lower().endswith("[bot]"):
return BOT, None
if not LOGIN_RE.match(login):
return NOT_FOUND, None
if gh.status(f"/orgs/{ORG}/blocks/{login}") == 204:
return BLOCKED, None
user = gh.get(f"/users/{login}")
if user is None:
return NOT_FOUND, None
Expand Down Expand Up @@ -110,13 +110,8 @@ def trusted(self, login, gh, now, excluded) -> bool:
if login.lower() not in {e.lower() for e in excluded} and LOGIN_RE.match(login):
e = self.get(login, now)
if e is not None:
if e.get("trusted") is not True:
return False
# A block added since caching must take effect now, not after the TTL.
if gh.status(f"/orgs/{ORG}/blocks/{login}") != 204:
return True
self.entries.pop(login.lower(), None)
return False
# A block added since caching is in excluded, so it took effect above.
return e.get("trusted") is True
reason, created = _check(login, gh, now, excluded)
if reason == BLOCKED:
# Not cached, so an unblock applies on the next run.
Expand All @@ -133,7 +128,7 @@ def account_age_days(self, login, now) -> int | None:
return None


def ensure_block_access(gh) -> None:
"""A token that can't read org blocks gets 404 per user, which would read as "not blocked"."""
if gh.get(f"/orgs/{ORG}/blocks?per_page=1") is None:
raise GitHubError(404, "token can't read org blocks")
def blocked_logins(gh) -> set[str]:
"""The org's blocked logins, lowercased: one listing per run instead of one lookup per account."""
# A token that can't read blocks gets 404, which must never read as "nobody blocked".
return {u["login"].lower() for u in gh.paginate(f"/orgs/{ORG}/blocks?per_page=100", must_exist=True)}
4 changes: 2 additions & 2 deletions scripts/reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from scripts.devstream.entries import DEFAULT_PATH, find, load, normalize_repo, save
from scripts.devstream.gh import GitHub
from scripts.devstream.issue_form import FormError, _repo, parse_report
from scripts.devstream.trust import ACTIVITY_QUERY, LOGIN_RE, check_user, ensure_block_access
from scripts.devstream.trust import ACTIVITY_QUERY, LOGIN_RE, blocked_logins, check_user
from scripts.validate_data import validate

REPO = "devstreamhq/devstream"
Expand Down Expand Up @@ -161,7 +161,7 @@ def handle_report(issue: dict, gh, entries: list, open_reports: list[dict],
# Before the trust lookups, so a flood costs one search per report.
if _other_recent_reports(login, issue["number"], gh, now) >= DAILY_LIMIT:
return _close(RATE_LIMIT)
ensure_block_access(gh)
excluded = set(excluded) | blocked_logins(gh)
if check_user(login, gh, now, excluded):
return _reply(UNTRUSTED, "needs-review")

Expand Down
28 changes: 24 additions & 4 deletions scripts/rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from scripts.devstream.license_detect import extra_licenses_for
from scripts.devstream.quality import (
MSG_GONE,
RepoResult,
MSG_LICENSE,
MSG_NO_README,
MSG_SHORT_README,
Expand All @@ -37,7 +38,8 @@
# An older sighting is a different episode, so it starts the week of grace again.
STALE = timedelta(days=14)
RENAME_KEEP = timedelta(days=8)
REMOVAL_REASONS = (MSG_GONE, MSG_LICENSE, MSG_NO_README, MSG_SHORT_README)
MSG_REPLACED = "A different repo now lives at this name, not the one that was listed."
REMOVAL_REASONS = (MSG_GONE, MSG_REPLACED, MSG_LICENSE, MSG_NO_README, MSG_SHORT_README)
REPO_INPUT = re.compile(r"[A-Za-z0-9-]{1,39}/[A-Za-z0-9._-]{1,100}")
API = "/repos/devstreamhq/devstream"
BOT = "devstreamhq-bot[bot]"
Expand All @@ -48,6 +50,14 @@
"The weekly health check couldn't decide on `{repo}` for {weeks} weeks running, "
"so it isn't being warned or removed. Could someone take a look?\n\nLast error: `{error}`\n"
)
REPLACED_TITLE = "A different repo now answers to {repo}"
REPLACED_BODY = (
"`{repo}` was listed as GitHub repo {want}, but that name now belongs to repo {got}. "
"Usually that means the owner renamed or deleted their account and someone else took the name.\n\n"
"The entry has a removal notice, and the hourly Likes sync keeps its old card and holds it out of Trending. "
"It still links to that name until it comes off the list, "
"so could someone check which repo is the real one before then?\n"
)

WARN = (
"Heads up: `{repo}` moves to the archive on {date} unless something changes. "
Expand Down Expand Up @@ -211,9 +221,17 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict:
rot_state = rot_state or {}
results = {}
undecided = {}
replaced = []
for e in targets:
try:
results[e["repo"]] = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"]))
r = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"]))
if e.get("repo_id") is None:
logging.warning("%s has no repo_id, so a replaced repo can't be caught", e["repo"])
elif r.exists and r.repo_id is not None and r.repo_id != e["repo_id"]:
# Someone else's repo at the listed name inherits nothing: it reads as gone, and a person decides.
replaced.append({"repo": e["repo"], "kind": "replaced", "want": e["repo_id"], "got": r.repo_id})
r = RepoResult(False, e["repo"], [MSG_REPLACED], "remove")
results[e["repo"]] = r
except GitHubError as exc:
# An owner's IP allow list hides the repo from our token; that says nothing about the repo.
if not isinstance(exc, Undecided) and not (exc.status == 403 and "IP allow list" in exc.message):
Expand All @@ -227,6 +245,7 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict:
except Exception:
# The weeks are only a review prompt; failing to record them mustn't stop the check.
logging.exception("could not record undecided repos")
review += replaced
new, actions, rot = plan_rot(entries, results, now, rot_state)
changed = new != sorted(entries, key=sort_key)
if changed:
Expand Down Expand Up @@ -272,10 +291,11 @@ def open_reviews(gh, review: list[dict]) -> None:
for r in review:
if not REPO_INPUT.fullmatch(r["repo"]):
raise ValueError(f"not an owner/name: {r['repo']!r}")
title = REVIEW_TITLE.format(repo=r["repo"])
title, body = (REPLACED_TITLE, REPLACED_BODY) if r.get("kind") == "replaced" else (REVIEW_TITLE, REVIEW_BODY)
title = title.format(repo=r["repo"])
if title in open_titles:
continue
gh.post(f"{API}/issues", {"title": title, "body": REVIEW_BODY.format(**r), "labels": ["needs-review"]})
gh.post(f"{API}/issues", {"title": title, "body": body.format(**r), "labels": ["needs-review"]})


def pr_body(actions) -> str:
Expand Down
29 changes: 16 additions & 13 deletions scripts/submit.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from scripts.devstream.issue_form import FormError, parse_submission
from scripts.devstream.license_detect import extra_licenses_for
from scripts.devstream.quality import check_repo
from scripts.devstream.trust import check_user, ensure_block_access
from scripts.devstream.trust import blocked_logins, check_user
from scripts.validate_data import validate

REPO = "devstreamhq/devstream"
Expand Down Expand Up @@ -71,8 +71,7 @@ def decide(issue: dict, gh, now: datetime, entries: list[Entry], excluded: set[s
except FormError as e:
return Decision(False, [str(e)])

ensure_block_access(gh)
reason = check_user(login, gh, now, excluded)
reason = check_user(login, gh, now, excluded | blocked_logins(gh))
if reason:
return Decision(False, [reason])

Expand All @@ -81,18 +80,20 @@ def decide(issue: dict, gh, now: datetime, entries: list[Entry], excluded: set[s
if _recent_submissions(login, gh, now) > WEEKLY_LIMIT:
reasons.append(RATE_LIMIT)

more, canonical, description = _check(form["repository"], form["description"], gh, now, entries, form["category"])
more, canonical, description, repo_id = _check(form["repository"], form["description"], gh, now, entries,
form["category"])
reasons += more
if reasons:
return Decision(False, reasons)
return Decision(True, [], _entry(canonical, form["category"], description, login, now, issue["number"]))
return Decision(True, [], _entry(canonical, repo_id, form["category"], description, login, now, issue["number"]))


def _check(link: str, text: str, gh, now: datetime, entries: list[Entry], category: str) -> tuple[list[str], str, str | None]:
"""The repo and description checks: (reasons, canonical repo, cleaned description)."""
def _check(link: str, text: str, gh, now: datetime, entries: list[Entry],
category: str) -> tuple[list[str], str, str | None, int | None]:
"""The repo and description checks: (reasons, canonical repo, cleaned description, repo ID)."""
dup = find(entries, link)
if dup:
return [_dup_reason(dup)], link, None
return [_dup_reason(dup)], link, None, None

reasons = []
result = check_repo(link, gh, now, "submit", allowed_extra=extra_licenses_for(category))
Expand All @@ -101,21 +102,22 @@ def _check(link: str, text: str, gh, now: datetime, entries: list[Entry], catego
else:
dup = find(entries, result.canonical)
if dup:
return [_dup_reason(dup)], result.canonical, None
return [_dup_reason(dup)], result.canonical, None, None
reasons += result.failures

description, why = clean_description(text)
if description is None and result.meta.get("description"):
description = clean_description(result.meta["description"])[0]
if description is None:
reasons.append(why)
return reasons, result.canonical, description
return reasons, result.canonical, description, result.repo_id


def _entry(repo: str, category: str, description: str, login: str, now: datetime, issue: int,
def _entry(repo: str, repo_id: int, category: str, description: str, login: str, now: datetime, issue: int,
seed: bool = False) -> Entry:
return {
"repo": repo,
"repo_id": repo_id,
"category": category,
"description": description,
"submitted_by": login,
Expand Down Expand Up @@ -243,15 +245,16 @@ def _seed_row(row: dict, gh, now: datetime, entries: list[Entry], seen: set[str]
out["reasons"] = [f"unknown category {row['category']!r}"]
return out
try:
reasons, canonical, description = _check(row["repo"], row["description"], gh, now, entries, row["category"])
reasons, canonical, description, repo_id = _check(row["repo"], row["description"], gh, now, entries,
row["category"])
except (GitHubError, RateLimited, OSError) as e:
# Undecided is a GitHubError; none of these say anything about the repo.
return out | {"status": "undecided", "reasons": [str(e)]}
if normalize_repo(canonical) in seen:
reasons = [SEED_DUP]
if reasons:
return out | {"reasons": reasons}
return out | {"status": "ok", "entry": _entry(canonical, row["category"], description, SEED_BY, now,
return out | {"status": "ok", "entry": _entry(canonical, repo_id, row["category"], description, SEED_BY, now,
SEED_ISSUE, seed=True)}


Expand Down
19 changes: 14 additions & 5 deletions scripts/sync_likes.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
rank_entries,
)
from scripts.devstream.stats import API, delete_file, ensure_branch, list_dir, read_json, update_json
from scripts.devstream.trust import TrustCache, ensure_block_access
from scripts.devstream.trust import TrustCache, blocked_logins

# Flagged entries stay listed and ranked; only Trending hides them.
RANKED = {"live", "warned", "flagged"}
Expand Down Expand Up @@ -230,18 +230,27 @@ def run(gh, entries, now: datetime, excluded: set[str]) -> None:
prev_meta = read_json(gh, "meta.json")[0]
# A repo GitHub hides from us this hour keeps its last card rather than losing stars and avatar.
listed = set(keys)
meta = {k: v for k, v in (prev_meta or {}).items() if k in listed} | fetch_meta(gh, keys)
fetched = fetch_meta(gh, keys)
replaced = set()
for k, m in list(fetched.items()):
got, want = m.pop("repo_id"), ranked[k].get("repo_id")
if want is not None and got != want:
# Someone else's repo at the listed name: its stars and avatar never reach the card or Trending.
logging.warning("%s now names repo %s, not the listed %s; keeping its last card", ranked[k]["repo"], got, want)
del fetched[k]
replaced.add(k)
meta = {k: v for k, v in (prev_meta or {}).items() if k in listed} | fetched
reactors = fetch_reactors(gh, sorted({mapping[k] for k in keys if k in mapping}))
prev_flags = read_json(gh, "flags.json")[0]
flags = copy.deepcopy(prev_flags or {})
_review(gh, flags, now)

cache = _load_cache(read_json(gh, "trust-cache.json")[0])
ensure_block_access(gh)
excluded = excluded | blocked_logins(gh)
seen = {}

def trusted(login):
# One check per run; the cache re-checks blocks on every call otherwise.
# One check per login per run.
if login not in seen:
seen[login] = cache.trusted(login, gh, now, excluded)
return seen[login]
Expand Down Expand Up @@ -283,7 +292,7 @@ def trusted(login):
"delta_7d": likes[k] - base,
"rank": ranks[k],
"rank_delta_7d": o["rank"] - ranks[k] if o else 0,
"flagged": ranked[k]["status"] == "flagged" or k in bursts or _under_review(flags, k),
"flagged": ranked[k]["status"] == "flagged" or k in bursts or k in replaced or _under_review(flags, k),
}

snap_path = f"snapshots/{now:%Y-%m-%d}.json"
Expand Down
8 changes: 6 additions & 2 deletions scripts/tests/fake_gh.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ class FakeGitHub:
Only .get returns None on a missing route; post, patch, put and delete raise
GitHubError(404). A route whose value is None answers a write like a
204. paginate yields nothing for a missing route (the real client does
the same for a 404 on the first page; the fake has a single page).
the same for a 404 on the first page, unless must_exist; the fake has a single page).

.last_redirect is reset at the start of every call.

Expand Down Expand Up @@ -98,9 +98,13 @@ def status(self, path) -> int:
return v.code
return 404 if v is None else 200

def paginate(self, path, params=None):
def paginate(self, path, params=None, must_exist=False):
self.last_redirect = None
page = self._find("GET", path, params, None)
if isinstance(page, Status) and self._ok(page):
page = None
if page is None and must_exist:
raise GitHubError(404, f"page not found: {path}")
if isinstance(page, dict) and "items" in page:
page = page["items"]
yield from page or []
Expand Down
Loading
Loading