Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions scripts/devstream/description.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
FORBIDDEN = set("<>[]()`*_#")
EMOJI = re.compile("[\U0001f000-\U0001faff\u2600-\u27bf]")
VARIATION = re.compile("[\ufe00-\ufe0f]")
ZWJ = "\u200d"
# A zero-width joiner between two emoji builds one emoji (a family, say), so it survives the stripping.
EMOJI_ZWJ = re.compile("(?<=[\U0001f000-\U0001faff\u2600-\u27bf\ufe00-\ufe0f])\u200d(?=[\U0001f000-\U0001faff\u2600-\u27bf])")
# Only names that are not real TLDs a spammer would use; Node.js and Vue.js rely on "js".
NOT_TLDS = {"js", "ts", "py", "md", "rb"}
DOMAIN = re.compile(r"(?<![\w.-])(?:[a-z0-9-]+\.)+([a-z]{2,24})(?![a-z0-9])", re.I)
Expand All @@ -26,7 +29,8 @@

def _strip_invisible(s: str) -> str:
# Cf also covers format characters outside the table, such as U+0600-0605.
return IGNORABLE.sub("", "".join(c for c in s if unicodedata.category(c) != "Cf"))
return ZWJ.join(IGNORABLE.sub("", "".join(c for c in part if unicodedata.category(c) != "Cf"))
for part in EMOJI_ZWJ.split(s))


def _load_blocklist() -> list[str]:
Expand Down Expand Up @@ -69,10 +73,13 @@ def _has_url(t: str) -> bool:


def _emoji_run_too_long(t: str) -> bool:
run = 0
run, joined = 0, False
for ch in t:
if EMOJI.match(ch):
run += 1
if ch == ZWJ: # the next emoji is part of this one
joined = True
elif EMOJI.match(ch):
run += not joined
joined = False
if run > 2:
return True
elif not VARIATION.match(ch): # variation selectors ride along with their emoji
Expand All @@ -89,7 +96,9 @@ def clean_description(text: str) -> tuple[str | None, str | None]:
if _emoji_run_too_long(t):
return None, "Descriptions can't have more than 2 emoji in a row."
t = " ".join(VARIATION.sub("", t).split())
final = t.rstrip("!?,;: ").rstrip(".") + "."
body = t.rstrip("!?,;: ")
# A question keeps one question mark; anything else ends in a full stop.
final = body.rstrip(".") + ("?" if "?" in t[len(body):] else ".")
# Both limits cover the finished text: a 140-character input with no period
# would be stored at 141, and "Toolkit!!!" shrinks to an 8-character "Toolkit.".
if not MIN_LEN <= len(final) <= MAX_LEN:
Expand Down
13 changes: 11 additions & 2 deletions scripts/devstream/issue_form.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
CHECKS = ("i read the quality bar", "this repo is open source")
BAD_REPO = "That doesn't look like a GitHub repo link."
INCOMPLETE = "The form looks incomplete. Please open a new submission and fill in every field."
HEADING_IN_DESCRIPTION = (
"Your description has a line starting with ###, which cuts it short. Please remove it and submit again."
)
REPO_RE = re.compile(r"^[A-Za-z0-9-]{1,39}/[A-Za-z0-9._-]{1,100}$")
HOST_RE = re.compile(r"^(?:https?://)?(?:www\.)?github\.com/", re.I)

Expand All @@ -30,9 +33,10 @@ def _repo(raw: str) -> str:
raise FormError(BAD_REPO)
if len(parts) < 2:
raise FormError(BAD_REPO)
owner, name = parts[0], parts[1].removesuffix(".git")
owner, name = parts[0], re.sub(r"\.git$", "", parts[1], flags=re.I)
cand = f"{owner}/{name}"
if not REPO_RE.match(cand) or not name.strip("."):
# GitHub drops a .git suffix from new repo names, so a second one is never a real repo.
if not REPO_RE.match(cand) or not name.strip(".") or name.lower().endswith(".git"):
raise FormError(BAD_REPO)
return normalize_repo(cand)

Expand Down Expand Up @@ -60,6 +64,11 @@ def parse_submission(body: str) -> dict[str, str]:
for k in HEADINGS.values():
if not sections.get(k):
raise FormError(INCOMPLETE)
# The form puts Checklist right after Description; any other heading there was typed into the answer.
heads = [h.strip().lower() for h in re.findall(r"^### (.+?)\s*$", body.replace("\r\n", "\n"), re.M)]
i = heads.index("description")
if heads[i + 1:i + 2] != ["checklist"]:
raise FormError(HEADING_IN_DESCRIPTION)

checked = {
m.group(1).strip().lower()
Expand Down
17 changes: 17 additions & 0 deletions scripts/tests/test_description.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,23 @@ def test_emoji_run_with_variation_selector(self):
# Any selector, not only U+FE0F: they are all dropped before storing.
bad("Stars \u2728\u2728\ufe08\u2728\u2728 everywhere here")

def test_zwj_emoji_counts_once(self):
family = "\U0001F468\u200d\U0001F469\u200d\U0001F467"
self.assertEqual(ok(f"Plans for the whole {family} at once"), f"Plans for the whole {family} at once.")
ok(f"Plans {family}{family} for everyone here")
bad(f"Plans {family}{family}{family} for everyone here")
# Still stripped anywhere else, so it can't split a blocked word or hide in text.
self.assertEqual(ok("A tool\u200d that works well"), "A tool that works well.")
# Re-checking the stored text gives the same answer, as validate_data does.
out = ok(f"Plans for the whole {family} at once")
self.assertEqual(clean(out), (out, None))

def test_question_keeps_one_question_mark(self):
self.assertEqual(ok("Why write YAML by hand?!?!"), "Why write YAML by hand?")
self.assertEqual(ok("Why write YAML by hand?"), "Why write YAML by hand?")
self.assertEqual(ok("A tool that works well!!!"), "A tool that works well.")
self.assertEqual(clean("Why write YAML by hand?"), ("Why write YAML by hand?", None))

def test_removed_terms_allowed(self):
ok("Work from home setup scripts for devs")
ok("A bulk sms sender for alert systems")
Expand Down
10 changes: 10 additions & 0 deletions scripts/tests/test_issue_form.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ def test_bad_repository_rejected(self):
"o/r r",
"-/..",
"a" * 40 + "/r",
"https://github.com/o/r.git.git",
"o/r.GIT.git",
):
with self.assertRaises(FormError, msg=r) as cm:
parse_submission(body(repo=r))
Expand All @@ -85,6 +87,14 @@ def test_extra_and_duplicate_headings(self):
b = body() + "\n### Other\n\nhi\n\n### Description\n\nsecond\n"
self.assertEqual(parse_submission(b)["description"], "text")

def test_heading_inside_description_is_reported(self):
for desc in ("A fine tool\n\n### Repository\n\nmore", "A fine tool\n### Notes\nmore"):
with self.assertRaises(FormError) as cm:
parse_submission(body(desc=desc))
self.assertIn("line starting with ###", str(cm.exception))
with self.assertRaises(FormError):
parse_submission(body(desc=desc).replace("\n", "\r\n"))

def test_crlf(self):
self.assertEqual(parse_submission(body().replace("\n", "\r\n"))["repository"], "o/r")

Expand Down
Loading