From a9ecd071b76ead1876f3cb12d514133ec1f338cc Mon Sep 17 00:00:00 2001 From: Foothills Digital Date: Fri, 6 Nov 2026 08:23:07 -0700 Subject: [PATCH] fix(submit): handle emoji sequences, ?!, .git.git and ### in descriptions A zero-width joiner between two emoji now survives the invisible- character strip and the joined sequence counts as one emoji, so a family emoji no longer counts as three in a row. A trailing run of punctuation that has a question mark now ends in one ?, so a question stays a question. Anything else still ends in a full stop. A repo link ending in .git.git is rejected; GitHub never names a repo that way. A ### line typed into the description used to cut it short without a word. The submitter now gets an error that says so. --- scripts/devstream/description.py | 19 ++++++++++++++----- scripts/devstream/issue_form.py | 13 +++++++++++-- scripts/tests/test_description.py | 17 +++++++++++++++++ scripts/tests/test_issue_form.py | 10 ++++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/scripts/devstream/description.py b/scripts/devstream/description.py index dd5aae2..1d1e565 100644 --- a/scripts/devstream/description.py +++ b/scripts/devstream/description.py @@ -6,6 +6,9 @@ FORBIDDEN = set("<>[]()`*_#") EMOJI = re.compile("[\U0001f000-\U0001faff\u2600-\u27bf]") VARIATION = re.compile("[\ufe00-\ufe0f]") +ZWJ = "\u200d" +# A zero-width joiner between two emoji builds one emoji (a family, say), so it survives the stripping. +EMOJI_ZWJ = re.compile("(?<=[\U0001f000-\U0001faff\u2600-\u27bf\ufe00-\ufe0f])\u200d(?=[\U0001f000-\U0001faff\u2600-\u27bf])") # Only names that are not real TLDs a spammer would use; Node.js and Vue.js rely on "js". NOT_TLDS = {"js", "ts", "py", "md", "rb"} DOMAIN = re.compile(r"(? str: # Cf also covers format characters outside the table, such as U+0600-0605. - return IGNORABLE.sub("", "".join(c for c in s if unicodedata.category(c) != "Cf")) + return ZWJ.join(IGNORABLE.sub("", "".join(c for c in part if unicodedata.category(c) != "Cf")) + for part in EMOJI_ZWJ.split(s)) def _load_blocklist() -> list[str]: @@ -69,10 +73,13 @@ def _has_url(t: str) -> bool: def _emoji_run_too_long(t: str) -> bool: - run = 0 + run, joined = 0, False for ch in t: - if EMOJI.match(ch): - run += 1 + if ch == ZWJ: # the next emoji is part of this one + joined = True + elif EMOJI.match(ch): + run += not joined + joined = False if run > 2: return True elif not VARIATION.match(ch): # variation selectors ride along with their emoji @@ -89,7 +96,9 @@ def clean_description(text: str) -> tuple[str | None, str | None]: if _emoji_run_too_long(t): return None, "Descriptions can't have more than 2 emoji in a row." t = " ".join(VARIATION.sub("", t).split()) - final = t.rstrip("!?,;: ").rstrip(".") + "." + body = t.rstrip("!?,;: ") + # A question keeps one question mark; anything else ends in a full stop. + final = body.rstrip(".") + ("?" if "?" in t[len(body):] else ".") # Both limits cover the finished text: a 140-character input with no period # would be stored at 141, and "Toolkit!!!" shrinks to an 8-character "Toolkit.". if not MIN_LEN <= len(final) <= MAX_LEN: diff --git a/scripts/devstream/issue_form.py b/scripts/devstream/issue_form.py index cf57091..7f1f972 100644 --- a/scripts/devstream/issue_form.py +++ b/scripts/devstream/issue_form.py @@ -12,6 +12,9 @@ CHECKS = ("i read the quality bar", "this repo is open source") BAD_REPO = "That doesn't look like a GitHub repo link." INCOMPLETE = "The form looks incomplete. Please open a new submission and fill in every field." +HEADING_IN_DESCRIPTION = ( + "Your description has a line starting with ###, which cuts it short. Please remove it and submit again." +) REPO_RE = re.compile(r"^[A-Za-z0-9-]{1,39}/[A-Za-z0-9._-]{1,100}$") HOST_RE = re.compile(r"^(?:https?://)?(?:www\.)?github\.com/", re.I) @@ -30,9 +33,10 @@ def _repo(raw: str) -> str: raise FormError(BAD_REPO) if len(parts) < 2: raise FormError(BAD_REPO) - owner, name = parts[0], parts[1].removesuffix(".git") + owner, name = parts[0], re.sub(r"\.git$", "", parts[1], flags=re.I) cand = f"{owner}/{name}" - if not REPO_RE.match(cand) or not name.strip("."): + # GitHub drops a .git suffix from new repo names, so a second one is never a real repo. + if not REPO_RE.match(cand) or not name.strip(".") or name.lower().endswith(".git"): raise FormError(BAD_REPO) return normalize_repo(cand) @@ -60,6 +64,11 @@ def parse_submission(body: str) -> dict[str, str]: for k in HEADINGS.values(): if not sections.get(k): raise FormError(INCOMPLETE) + # The form puts Checklist right after Description; any other heading there was typed into the answer. + heads = [h.strip().lower() for h in re.findall(r"^### (.+?)\s*$", body.replace("\r\n", "\n"), re.M)] + i = heads.index("description") + if heads[i + 1:i + 2] != ["checklist"]: + raise FormError(HEADING_IN_DESCRIPTION) checked = { m.group(1).strip().lower() diff --git a/scripts/tests/test_description.py b/scripts/tests/test_description.py index 475bcd7..24df6ef 100644 --- a/scripts/tests/test_description.py +++ b/scripts/tests/test_description.py @@ -185,6 +185,23 @@ def test_emoji_run_with_variation_selector(self): # Any selector, not only U+FE0F: they are all dropped before storing. bad("Stars \u2728\u2728\ufe08\u2728\u2728 everywhere here") + def test_zwj_emoji_counts_once(self): + family = "\U0001F468\u200d\U0001F469\u200d\U0001F467" + self.assertEqual(ok(f"Plans for the whole {family} at once"), f"Plans for the whole {family} at once.") + ok(f"Plans {family}{family} for everyone here") + bad(f"Plans {family}{family}{family} for everyone here") + # Still stripped anywhere else, so it can't split a blocked word or hide in text. + self.assertEqual(ok("A tool\u200d that works well"), "A tool that works well.") + # Re-checking the stored text gives the same answer, as validate_data does. + out = ok(f"Plans for the whole {family} at once") + self.assertEqual(clean(out), (out, None)) + + def test_question_keeps_one_question_mark(self): + self.assertEqual(ok("Why write YAML by hand?!?!"), "Why write YAML by hand?") + self.assertEqual(ok("Why write YAML by hand?"), "Why write YAML by hand?") + self.assertEqual(ok("A tool that works well!!!"), "A tool that works well.") + self.assertEqual(clean("Why write YAML by hand?"), ("Why write YAML by hand?", None)) + def test_removed_terms_allowed(self): ok("Work from home setup scripts for devs") ok("A bulk sms sender for alert systems") diff --git a/scripts/tests/test_issue_form.py b/scripts/tests/test_issue_form.py index 08ff9e3..aca69ec 100644 --- a/scripts/tests/test_issue_form.py +++ b/scripts/tests/test_issue_form.py @@ -68,6 +68,8 @@ def test_bad_repository_rejected(self): "o/r r", "-/..", "a" * 40 + "/r", + "https://github.com/o/r.git.git", + "o/r.GIT.git", ): with self.assertRaises(FormError, msg=r) as cm: parse_submission(body(repo=r)) @@ -85,6 +87,14 @@ def test_extra_and_duplicate_headings(self): b = body() + "\n### Other\n\nhi\n\n### Description\n\nsecond\n" self.assertEqual(parse_submission(b)["description"], "text") + def test_heading_inside_description_is_reported(self): + for desc in ("A fine tool\n\n### Repository\n\nmore", "A fine tool\n### Notes\nmore"): + with self.assertRaises(FormError) as cm: + parse_submission(body(desc=desc)) + self.assertIn("line starting with ###", str(cm.exception)) + with self.assertRaises(FormError): + parse_submission(body(desc=desc).replace("\n", "\r\n")) + def test_crlf(self): self.assertEqual(parse_submission(body().replace("\n", "\r\n"))["repository"], "o/r")