Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 45 additions & 17 deletions scripts/devstream/license_detect.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
from scripts.devstream.gh import GitHubError

OSI = frozenset(json.loads((Path(__file__).parents[2] / "data" / "osi-licenses.json").read_text()))
# Content licenses, for categories whose entries are text rather than code.
CONTENT_LICENSES = frozenset({"CC0-1.0", "CC-BY-4.0", "CC-BY-SA-4.0"})
LICENSE_RE = re.compile(r"^(licen[cs]e|copying|unlicense)([-._][\w.-]+)?(\.(md|txt|rst))?$", re.I)
# Narrower than LICENSE_RE: these stems also name source files (copyright.py, gpl.js).
OTHER_RE = re.compile(r"^(copyright|artistic|bsdl|gpl)(\.(md|txt|rst)|-[\w-]+|-([\w-]*-)?\d+(\.\d+)*(\.(md|txt|rst))?)?$", re.I)
Expand Down Expand Up @@ -62,6 +64,10 @@ def match(name):
("Unlicense", ["this is free and unencumbered software released into the public domain", "unlicense.org"], []),
("Zlib", ["this software is provided 'as-is'", "altered source versions must be plainly marked", "this notice may not be removed or altered from any source distribution"], []),
("EPL-2.0", ["eclipse public license", "2.0", "accompanying program is provided under the terms of this eclipse public license"], []),
# Only counted when allowed. NC and ND texts carry neither opening, so they never match.
("CC0-1.0", ["cc0 1.0 universal", "statement of purpose"], []),
("CC-BY-SA-4.0", ["attribution-sharealike 4.0 international", "sharealike"], []),
("CC-BY-4.0", ["attribution 4.0 international", "creative commons corporation"], []),
]

# README license names, lowercase.
Expand All @@ -75,10 +81,14 @@ def match(name):
"lgpl-3.0": "LGPL-3.0", "lgpl-2.1": "LGPL-2.1", "agpl-3.0": "AGPL-3.0",
"mpl 2.0": "MPL-2.0", "mpl-2.0": "MPL-2.0", "isc": "ISC",
"unlicense": "Unlicense", "the unlicense": "Unlicense",
"cc0": "CC0-1.0", "cc0-1.0": "CC0-1.0", "cc by 4.0": "CC-BY-4.0", "cc-by-4.0": "CC-BY-4.0",
"creative commons attribution 4.0": "CC-BY-4.0", "cc by-sa 4.0": "CC-BY-SA-4.0",
}
NAME_RE = re.compile(
r"(?<![\w-])(" + "|".join(re.escape(n) for n in sorted(NAMES, key=len, reverse=True)) + r")(?![\w-])"
)
# A CC name next to one of these is a restricted variant (or a mix), so it is not a match.
NC_ND = frozenset({"nc", "noncommercial", "non", "nd", "noderivatives"})
NEGATION_RE = re.compile(r"\bnot\b|isn't|\bno longer\b|\bformerly\b")
HEADING_RE = re.compile(r"\s{0,3}(#{1,6})\s+(.*)")
SPDX_MAX_LINES = 10
Expand All @@ -88,6 +98,10 @@ def match(name):
GENERIC = "a non-open-source license"


def extra_licenses_for(category: str) -> frozenset:
return CONTENT_LICENSES if category == "learning-resources" else frozenset()


class Undecided(GitHubError):
pass

Expand All @@ -101,15 +115,15 @@ def _non_osi(text: str, markers=NON_OSI) -> str | None:
return next((name for marker, name in markers if marker.lower() in t), None)


def _id_ok(i: str) -> bool:
return i in OSI or (i.endswith("+") and i[:-1] in OSI)
def _id_ok(i: str, allowed: frozenset = frozenset()) -> bool:
return i in OSI or i in allowed or (i.endswith("+") and i[:-1] in OSI)


class _Bad(Exception):
pass


def _spdx_eval(expr: str) -> tuple[bool, str | None]:
def _spdx_eval(expr: str, allowed: frozenset = frozenset()) -> tuple[bool, str | None]:
"""(is OSI, first non-OSI ID). Raises _Bad on anything unparseable."""
toks = expr.replace("(", " ( ").replace(")", " ) ").split()
pos = 0
Expand Down Expand Up @@ -146,7 +160,7 @@ def atom():
if peek() in (None, "AND", "OR", "WITH", "(", ")"):
raise _Bad
take()
ok = _id_ok(t)
ok = _id_ok(t, allowed)
if not ok and bad is None:
bad = t
return ok
Expand All @@ -171,7 +185,7 @@ def or_():
return result, bad


def _spdx_scan(text: str) -> tuple[str | None, str | None]:
def _spdx_scan(text: str, allowed: frozenset = frozenset()) -> tuple[str | None, str | None]:
"""(osi expression, non-OSI name) from SPDX-License-Identifier lines."""
found = bad = None
seen = 0
Expand All @@ -186,7 +200,7 @@ def _spdx_scan(text: str) -> tuple[str | None, str | None]:
try:
if len(expr) > SPDX_MAX_CHARS:
raise _Bad
ok, name = _spdx_eval(expr)
ok, name = _spdx_eval(expr, allowed)
except (_Bad, RecursionError):
bad = bad or GENERIC
continue
Expand All @@ -197,12 +211,12 @@ def _spdx_scan(text: str) -> tuple[str | None, str | None]:
return found, bad


def _detect_text(text: str) -> tuple[str | None, str | None]:
def _detect_text(text: str, allowed: frozenset = frozenset()) -> tuple[str | None, str | None]:
"""(osi_id, non_osi_name) for one license file."""
bad = _non_osi(text)
if bad:
return None, bad
found, bad = _spdx_scan(text)
found, bad = _spdx_scan(text, allowed)
if bad:
return None, bad
if found:
Expand All @@ -211,12 +225,12 @@ def _detect_text(text: str) -> tuple[str | None, str | None]:
if t in ("mit", "mit license"):
return "MIT", None
for spdx, need, forbid in PHRASES:
if all(p in t for p in need) and not any(p in t for p in forbid):
if (spdx in OSI or spdx in allowed) and all(p in t for p in need) and not any(p in t for p in forbid):
return spdx, None
return None, None


def _readme_license(readme: str) -> tuple[str | None, str | None]:
def _readme_license(readme: str, allowed: frozenset = frozenset()) -> tuple[str | None, str | None]:
"""(osi_id, non_osi_name) from the README's License section."""
lines = readme.splitlines()
for n, line in enumerate(lines):
Expand All @@ -238,7 +252,8 @@ def _readme_license(readme: str) -> tuple[str | None, str | None]:
for ln in first:
low = _norm(ln)
m = NAME_RE.search(low)
if m and not NEGATION_RE.search(low):
if m and not NEGATION_RE.search(low) and not NC_ND.intersection(re.findall(r"[a-z0-9]+", low)) \
and _id_ok(NAMES[m.group(1)], allowed):
return NAMES[m.group(1)], None
return None, None

Expand All @@ -256,8 +271,11 @@ def _read_file(gh, r, path) -> str:
return raw[:MAX_BYTES].decode("utf-8", errors="replace")


def find_license(gh, r, readme_text: str | None) -> tuple[str | None, str | None]:
"""(osi_id, non_osi_name); (None, None) when nothing was found. API errors raise."""
def find_license(gh, r, readme_text: str | None, allowed: frozenset = frozenset()) -> tuple[str | None, str | None, bool]:
"""(osi_id, non_osi_name, cc_only); (None, None, False) when nothing was found. API errors raise.

cc_only: nothing acceptable here, but a content license is present that another category accepts.
"""
listing = gh.get(f"/repos/{r}/contents/")
if not isinstance(listing, list):
listing = []
Expand All @@ -268,13 +286,23 @@ def find_license(gh, r, readme_text: str | None) -> tuple[str | None, str | None
key=lambda e: (not LICENSE_RE.match(e[0]), e[0].lower(), e[0]),
)[:MAX_FILES]
if not entries:
return _readme_license(readme_text or "")
found, bad = _readme_license(readme_text or "", allowed)
cc_only = not (found or bad or allowed) and bool(_readme_license(readme_text or "", CONTENT_LICENSES)[0])
return found, bad, cc_only
# One read per file, in order, stopping at the first rejection so a later unreadable file can't hide it.
texts = []
found = None
for name, kind in entries:
if kind != "file":
continue # a symlink counts as a license file but isn't followed
osi, bad = _detect_text(_read_file(gh, r, name))
text = _read_file(gh, r, name)
texts.append(text)
osi, bad = _detect_text(text, allowed)
if bad:
return None, bad
return None, bad, False
found = found or osi
return found, None
if found or allowed:
return found, None, False
# Nothing here, so see whether a content license (already read) is what's present.
cc_only = any(_detect_text(t, CONTENT_LICENSES)[0] for t in texts)
return None, None, cc_only
16 changes: 10 additions & 6 deletions scripts/devstream/quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"We couldn't find an open-source license. If this is your repo, add a standard LICENSE file "
"(choosealicense.com has them) so GitHub can detect it, then submit again."
)
MSG_NOT_FOUND_CC = MSG_NOT_FOUND + " CC licenses are accepted for Learning & Resources only."
MSG_NO_README = "Add a README so people know what this is."
MSG_SHORT_README = "Your README needs a bit more detail (at least a short paragraph about what it does)."
MSG_FORK = "Forks need their own commits to be listed."
Expand Down Expand Up @@ -69,19 +70,21 @@ def _readme_text_length(text: str) -> int:
return len(re.sub(r"\s+", " ", text).strip())


def _license(repo, gh, r, readme_text) -> str | None:
def _license(repo, gh, r, readme_text, allowed=frozenset(), mode="submit") -> str | None:
"""None when fine, else the failure message."""
spdx = (repo.get("license") or {}).get("spdx_id")
if spdx == "NOASSERTION":
spdx = ((gh.get(f"/repos/{r}/license") or {}).get("license") or {}).get("spdx_id")
if spdx not in NO_LICENSE:
return None if spdx in OSI else MSG_LICENSE
return None if spdx in OSI or spdx in allowed else MSG_LICENSE
if spdx == "NONE":
return MSG_LICENSE
found, bad = find_license(gh, r, readme_text)
found, bad, cc_only = find_license(gh, r, readme_text, allowed)
if found:
return None
return MSG_NON_OSI.format(bad) if bad else MSG_NOT_FOUND
if bad:
return MSG_NON_OSI.format(bad)
return MSG_NOT_FOUND_CC if cc_only and mode == "submit" else MSG_NOT_FOUND


def _readme(gh, r) -> tuple[str | None, int] | None:
Expand Down Expand Up @@ -149,7 +152,8 @@ def _old_enough(created_at, now) -> bool:
return False


def check_repo(full: str, gh, now: datetime, mode: Literal["submit", "rot"]) -> RepoResult:
def check_repo(full: str, gh, now: datetime, mode: Literal["submit", "rot"],
allowed_extra: frozenset = frozenset()) -> RepoResult:
repo = gh.get(f"/repos/{full}")
if repo is None:
return RepoResult(False, full, [MSG_GONE], "remove")
Expand All @@ -169,7 +173,7 @@ def check_repo(full: str, gh, now: datetime, mode: Literal["submit", "rot"]) ->
raise Undecided(0, f"{r} has no pushed_at")
age = None if pushed is None else now - datetime.fromisoformat(pushed)
readme = _readme(gh, r)
license_msg = _license(repo, gh, r, readme[0] if readme else None)
license_msg = _license(repo, gh, r, readme[0] if readme else None, allowed_extra, mode)
license_ok = license_msg is None
readme_ok = readme is not None and readme[1] >= README_MIN
text = (readme[0] if readme else None) or ""
Expand Down
3 changes: 2 additions & 1 deletion scripts/rot.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from scripts.devstream.discussions import REMOVED_PREFIX
from scripts.devstream.entries import DEFAULT_PATH, find, load, normalize_repo, save, sort_key
from scripts.devstream.gh import GitHub
from scripts.devstream.license_detect import extra_licenses_for
from scripts.devstream.quality import (
MSG_GONE,
MSG_LICENSE,
Expand Down Expand Up @@ -200,7 +201,7 @@ def prepare(gh, now, path=DEFAULT_PATH, repo=None, check=check_repo) -> dict:
results = {}
for e in targets:
try:
results[e["repo"]] = check(e["repo"], gh, now, "rot")
results[e["repo"]] = check(e["repo"], gh, now, "rot", allowed_extra=extra_licenses_for(e["category"]))
except Undecided as exc:
logging.warning("skipping %s: %s", e["repo"], exc)
new, actions, rot = plan_rot(entries, results, now, rot_state)
Expand Down
9 changes: 5 additions & 4 deletions scripts/submit.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from scripts.devstream.entries import CATEGORIES, DEFAULT_PATH, Entry, find, load, normalize_repo, save
from scripts.devstream.gh import GitHub, GitHubError, RateLimited
from scripts.devstream.issue_form import FormError, parse_submission
from scripts.devstream.license_detect import extra_licenses_for
from scripts.devstream.quality import check_repo
from scripts.devstream.trust import check_user, ensure_block_access
from scripts.validate_data import validate
Expand Down Expand Up @@ -76,21 +77,21 @@ def decide(issue: dict, gh, now: datetime, entries: list[Entry], excluded: set[s
if _recent_submissions(login, gh, now) > WEEKLY_LIMIT:
reasons.append(RATE_LIMIT)

more, canonical, description = _check(form["repository"], form["description"], gh, now, entries)
more, canonical, description = _check(form["repository"], form["description"], gh, now, entries, form["category"])
reasons += more
if reasons:
return Decision(False, reasons)
return Decision(True, [], _entry(canonical, form["category"], description, login, now, issue["number"]))


def _check(link: str, text: str, gh, now: datetime, entries: list[Entry]) -> tuple[list[str], str, str | None]:
def _check(link: str, text: str, gh, now: datetime, entries: list[Entry], category: str) -> tuple[list[str], str, str | None]:
"""The repo and description checks: (reasons, canonical repo, cleaned description)."""
dup = find(entries, link)
if dup:
return [_dup_reason(dup)], link, None

reasons = []
result = check_repo(link, gh, now, "submit")
result = check_repo(link, gh, now, "submit", allowed_extra=extra_licenses_for(category))
if not result.exists:
reasons.append(NOT_FOUND)
else:
Expand Down Expand Up @@ -216,7 +217,7 @@ def _seed_row(row: dict, gh, now: datetime, entries: list[Entry], seen: set[str]
out["reasons"] = [f"unknown category {row['category']!r}"]
return out
try:
reasons, canonical, description = _check(row["repo"], row["description"], gh, now, entries)
reasons, canonical, description = _check(row["repo"], row["description"], gh, now, entries, row["category"])
except (GitHubError, RateLimited, OSError) as e:
# Undecided is a GitHubError; none of these say anything about the repo.
return out | {"status": "undecided", "reasons": [str(e)]}
Expand Down
Loading
Loading