diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0035be89..2d630027 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,16 +7,6 @@ permissions: pull-requests: read jobs: - # TODO: #572 - needs improvements - # build: - # runs-on: ubuntu-latest - # name: Check - # steps: - # - name: Checkout - # uses: actions/checkout@main - # - name: Check - # run: ./check.sh - markdownlint: runs-on: ubuntu-latest name: Markdownlint @@ -29,13 +19,11 @@ jobs: files: "*.md" config_file: .markdownlint.json - ensure_sorted: + check_format: runs-on: ubuntu-latest - permissions: - contents: read - name: Ensure sorted + name: Check format steps: - name: Checkout uses: actions/checkout@main - - name: Ensure sorted - run: python3 ensure_sorted.py + - name: Check format + run: python3 check_format.py diff --git a/check.sh b/check.sh deleted file mode 100755 index ffa9d1a3..00000000 --- a/check.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set extglob - -SOURCE_FILE=README.md - -FDROID_SETTINGS=$( \ - grep -ni "https://f-droid.org/app" ${SOURCE_FILE} \ - | sed "s/.*https:\/\/f-droid.org\/app\///g" \ - | sed "s/)].*/;/g" \ - ) - -FAILED=0 - -while IFS=';' read -ra TOK; do - for i in "${TOK[@]}"; do - echo -ne "Check '${i}' ... " - - if [[ "${i}" =~ ^([[:alnum:]]|[\._-])+$ ]]; - then - echo -e "\e[0;32m OK \e[0m" - else - echo -e "\e[0;31m INVALID \e[0m" - let "FAILED += 1" - fi - - done -done <<< "${FDROID_SETTINGS}" - -if [[ ${FAILED} -ge 1 ]]; -then - echo "Invalid Urls: ${FAILED}" - exit 1 -fi diff --git a/check_format.py b/check_format.py new file mode 100644 index 00000000..6769e827 --- /dev/null +++ b/check_format.py @@ -0,0 +1,370 @@ +"""Verify that the app entries in README.md match the formats + +Prints one ":: " per finding and exits with 1 if any +finding was produced, 2 if the file cannot be read or no known section was +recognized, and 0 otherwise. +""" + +import argparse +import re +import sys + +SECTIONS = ( + "App Store", + "Apps", + "Games", + "Rooting and Recovery", + "Tools", + "Tutorials and Guides", + "Getting more …", +) + +# Sections that are plain link lists rather than app entries: link targets +# may be relative (e.g. "doc/*.md") and one line may carry several links. +LINK_LIST_SECTIONS = ("Tutorials and Guides", "Getting more …") + +HEADING_RE = re.compile(r"^## – (.+?) –\s*$") +CATEGORY_RE = re.compile(r"^### • (.+?)\s*$") +SEPARATOR_RE = re.compile(r"^\s*-{3,}\s*$") + +# * [**Name**](URL) + everything that follows (store badges / garbage) +ENTRY_HEAD_RE = re.compile(r"^\s*\* \[\*\*(.+?)\*\*\]\(([^)]*)\)(.*)$") + +# An extra link on a link-list entry line: / [**Name**](URL) +SECOND_LINK_RE = re.compile(r"^ / \[\*\*(.+?)\*\*\]\(([^)]*)\)(.*)$") + +# The only acceptable badge markup (without leading space, matched from "") +CANON_BADGE_RE = re.compile( + r"^\*\*\[\[(F-Droid|IzzyOnDroid)\]\(([^)]*)\)\]\*\*" +) + +LINK_RE = re.compile(r"\[([^\[\]]*)\]\(([^)]*)\)") +STORE_URL_RE = { + "F-Droid": re.compile(r"^https://f-droid\.org/packages/(.+)$"), + "IzzyOnDroid": re.compile(r"^https://apt\.izzysoft\.de/packages/(.+)$"), +} +STORE_URL_HINT = { + "F-Droid": "https://f-droid.org/packages/", + "IzzyOnDroid": "https://apt.izzysoft.de/packages/", +} +APP_ID_RE = re.compile(r"^[A-Za-z0-9._-]+$") + + +def check_store_url(lineno, ctx, name, store, url): + """Return findings for the URL of a single store badge.""" + m = STORE_URL_RE[store].match(url) + if m is None: + return [ + ( + lineno, + ( + f"{ctx} [{name}]: invalid {store} URL '{url}' " + f"(must be exactly {STORE_URL_HINT[store]})" + ), + ) + ] + app_id = m.group(1) + if not APP_ID_RE.match(app_id): + return [ + ( + lineno, + ( + f"{ctx} [{name}]: invalid app id '{app_id}' in {store} link " + "(only letters, digits, '.', '_' and '-' are allowed)" + ), + ) + ] + return [] + + +def check_entry(lineno, line, section, ctx): + """Validate one entry line. Returns (findings, entry name or None).""" + findings = [] + m = ENTRY_HEAD_RE.match(line) + if m is None: + # main() only passes lines matching r"^\s*\* " to check_entry(). + rest = re.match(r"^\s*\* (.*)$", line).group(1).strip() + if rest.startswith("[**"): + findings.append( + ( + lineno, + f"{ctx}: malformed entry (expected '[**Name**](URL)'), got: {rest[:60]}", + ) + ) + else: + findings.append( + ( + lineno, + f"{ctx}: entry must start with '[**Name**](URL)', got: {rest[:60]}", + ) + ) + return findings, None + + name, url, tail = m.groups() + if not name.strip(): + findings.append((lineno, f"{ctx}: empty entry name")) + if section in LINK_LIST_SECTIONS: + if not url: + findings.append((lineno, f"{ctx} [{name}]: empty link target")) + elif not re.match(r"^https?://", url): + findings.append( + ( + lineno, + f"{ctx} [{name}]: project URL must be an absolute http(s) URL, got: '{url}'", + ) + ) + + # Link lists may carry extra " / [**Name**](URL)" links on one line. + if section in LINK_LIST_SECTIONS: + while True: + sm = SECOND_LINK_RE.match(tail) + if sm is None: + break + sub_name, sub_url, tail = sm.groups() + if not sub_name.strip(): + findings.append( + (lineno, f"{ctx} [{name}]: extra link has an empty name") + ) + if not sub_url: + findings.append( + (lineno, f"{ctx} [{name}]: extra link has an empty target") + ) + + pos = 0 + stores = [] + while True: + i = tail.find("", pos) + if i == -1: + break + + gap = tail[pos:i] + where = "before the store badges" if pos == 0 else "between store badges" + if gap.strip(): + findings.append( + ( + lineno, + f"{ctx} [{name}]: unexpected text {where}: '{gap.strip()[:60]}'", + ) + ) + elif gap != " ": + findings.append( + ( + lineno, + f"{ctx} [{name}]: expected exactly one space {where} (got {len(gap)})", + ) + ) + + cm = CANON_BADGE_RE.match(tail[i:]) + if cm: + store, badge_url = cm.groups() + findings.extend(check_store_url(lineno, ctx, name, store, badge_url)) + stores.append(store) + pos = i + cm.end() + continue + + # Non-canonical badge: salvage the store name/URL for a precise message. + inner = tail[i + len("") :] + close = inner.find("") + closed = close != -1 + content = inner[:close] if closed else inner + lm = LINK_RE.search(content) + store = lm.group(1) if lm else None + if store in ("F-Droid", "IzzyOnDroid"): + if not closed: + findings.append( + ( + lineno, + f"{ctx} [{name}]: {store} badge is not closed (missing '')", + ) + ) + elif not re.fullmatch(r"\*\*\[\[.*\]\(.*\)\]\*\*", content): + findings.append( + ( + lineno, + ( + f"{ctx} [{name}]: non-canonical {store} badge markup: '{tail[i : i + 70]}' " + f"(expected '**[[{store}](URL)]**')" + ), + ) + ) + if lm: + findings.extend(check_store_url(lineno, ctx, name, store, lm.group(2))) + stores.append(store) + elif store is None: + findings.append( + ( + lineno, + f"{ctx} [{name}]: unparseable store badge: '{tail[i : i + 70]}'", + ) + ) + else: + findings.append( + ( + lineno, + ( + f"{ctx} [{name}]: store badge '{store}' not allowed " + "(only 'F-Droid' and 'IzzyOnDroid' may be linked)" + ), + ) + ) + if closed: + pos = i + len("") + close + len("") + else: + pos = len(tail) + + leftover = tail[pos:] + if leftover.strip(): + findings.append( + ( + lineno, + f"{ctx} [{name}]: unexpected content after store badges: '{leftover.strip()[:60]}'", + ) + ) + elif leftover: + findings.append((lineno, f"{ctx} [{name}]: trailing whitespace on entry line")) + + allowed = [s for s in ("F-Droid", "IzzyOnDroid") if s in stores] + if stores and stores != allowed: + if len(stores) != len(set(stores)): + msg = f"duplicate store badge(s), found: {', '.join(stores)}" + else: + msg = f"store badges must appear with F-Droid before IzzyOnDroid, found: {', '.join(stores)}" + findings.append((lineno, f"{ctx} [{name}]: {msg}")) + + return findings, name + + +def check_sort(items, ctx): + """Return findings for every alphabetical inversion among top-level entries.""" + findings = [] + for idx in range(1, len(items)): + prev_line, prev_name = items[idx - 1] + this_line, this_name = items[idx] + if this_name.lower() < prev_name.lower(): + findings.append( + ( + this_line, + ( + f"{ctx}: entries not sorted alphabetically — '{this_name}' comes after '{prev_name}' " + f"(line {prev_line})" + ), + ) + ) + return findings + + +def main(): + parser = argparse.ArgumentParser( + description="Verify that the app entries in README.md match the formats from CONTRIBUTING.md.", + ) + parser.add_argument( + "readme", + nargs="?", + default="README.md", + help="path to the README (default: README.md)", + ) + parser.add_argument( + "--no-sort", action="store_true", help="skip the alphabetical-order check" + ) + args = parser.parse_args() + + try: + with open(args.readme, encoding="utf-8") as fh: + lines = fh.read().splitlines() + except OSError as exc: + print(f"error: cannot read {args.readme}: {exc}", file=sys.stderr) + return 2 + + section = None + category = None + items = [] # (lineno, name) of the top-level entries of the current category + findings = [] + entries_checked = 0 + sections_found = 0 + + def ctx(): + return section if category is None else f"{section} > {category}" + + def flush(): + if not args.no_sort and items: + findings.extend(check_sort(items, ctx())) + + for lineno, line in enumerate(lines, 1): + hm = HEADING_RE.match(line) + if hm: + flush() + heading = hm.group(1) + if heading in SECTIONS: + section = heading + sections_found += 1 + else: + section = None + findings.append( + ( + lineno, + f"unknown section '{heading}' (expected one of: {', '.join(SECTIONS)})", + ) + ) + category = None + items = [] + continue + cm = CATEGORY_RE.match(line) + if cm and section is not None: + flush() + category = cm.group(1) + items = [] + continue + if section is None: + continue + if not line.strip() or SEPARATOR_RE.match(line): + continue + if re.match(r"^\s*\* ", line): + top_level = line.startswith("*") + entry_findings, entry_name = check_entry(lineno, line, section, ctx()) + findings.extend(entry_findings) + entries_checked += 1 + if top_level and entry_name: + items.append((lineno, entry_name)) + else: + stripped = line.strip() + if "" in line: + findings.append( + ( + lineno, + ( + f"{ctx()}: store badge outside an entry line — entries must be " + f"'* [**Name**](URL) …', got: '{stripped[:60]}'" + ), + ) + ) + else: + findings.append( + ( + lineno, + f"{ctx()}: unexpected line in app section: '{stripped[:60]}'", + ) + ) + flush() + + if findings: + for ln, msg in sorted(findings): + print(f"{args.readme}:{ln}: {msg}") + print(f"\n{len(findings)} issue(s) found ({entries_checked} entries checked).") + return 1 + + if sections_found == 0: + print( + f"error: no known section found in {args.readme} " + f"(expected a heading like '## – {SECTIONS[0]} –')", + file=sys.stderr, + ) + return 2 + + print( + f"{args.readme}: OK — {entries_checked} entries match the CONTRIBUTING.md format." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/check_manually.sh b/check_manually.sh deleted file mode 100755 index 15e90fe0..00000000 --- a/check_manually.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash - -SOURCE_FILE=README.md - -function check_deprecated_repos() { - GITHUB_LINKS=$(grep -o "https://github.com/.*" "$SOURCE_FILE" \ - | sed 's|).*||') - - mapfile -t GITHUB_LINKS <<< "$GITHUB_LINKS" - - for link in "${GITHUB_LINKS[@]}"; do - echo "Checking if repo $link is not deprecated" - DEPRECATED=$(curl "$link" -s | awk -v depr_repo_res='false' \ - '/This repository has been archived by the owner. It is now read-only./ \ - {depr_repo_res="true"} END {print depr_repo_res}') - if [[ "$DEPRECATED" == "true" ]]; then - deprecated_repos+=("$link") - fi - done - - if [ -n "${deprecated_repos[*]}" ]; then - depr_repo_res=1 - fi -} - -function check_false_links() { - LINKS=$(grep -oP "http.*" "$SOURCE_FILE" \ - | sed -e "s|)$||" -e "s|).*||" \ - | grep -v img.shields.io \ - | grep -v travis-ci.org) - - mapfile -t LINKS <<< "$LINKS" - - for link in "${LINKS[@]}" - do - echo "Checking $link" - STATUS_CODE="$(curl -LI "$link" -o /dev/null -w '%{http_code}\n' -s)" - if [[ "$STATUS_CODE" != "200" ]] - then - bad_links+=("$link") - fi - done - - if [ -n "${bad_links[*]}" ] - then - link_res=1 - fi -} - -function check_links() { - local depr_repo_res=0 - local deprecated_repos - check_deprecated_repos - - local link_res=0 - local bad_links - check_false_links - - clear - if [[ link_res -eq "1" ]]; then - echo "Unreachable links were found:" - for link in "${bad_links[@]}"; do - echo "$link" - done - fi - - if [[ depr_repo_res -eq "1" ]]; then - echo "Deprecated repo were found:" - for link in "${deprecated_repos[@]}"; do - echo "$link" - done - fi - - if [[ depr_repo_res -eq 0 && link_res -eq 0 ]]; then - echo "All links are validated!" - fi -} - -check_links diff --git a/ensure_sorted.py b/ensure_sorted.py deleted file mode 100755 index a4a91aac..00000000 --- a/ensure_sorted.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/python3 -import re - -class bcolors: - HEADER = '\033[95m' - BLUE = '\033[94m' - CYAN = '\033[96m' - GREEN = '\033[92m' - YELLOW = '\033[93m' - RED = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - -class Category: - def __init__(self, name): - self.name = name - # a list of apps - self.apps = [] - - def add_app(self, app_str: str): - matches = re.findall("(?<=\\[\\*\\*).*(?=\\*\\*\\])", app_str) - if len(matches) != 1: - raise RuntimeError("These should be only one match") - app_name = matches[0] - # make it lower case and append it - self.apps.append(app_name.lower()) - - def is_sorted(self): - # I know not effiencent - return sorted(self.apps) == self.apps - - # Tell exactly where it's unsorted - def where_unsorted(self): - for i in range(1, len(self.apps)): - if self.apps[i] < self.apps[i-1]: - return f'App {bcolors.RED}{self.apps[i-1]}{bcolors.ENDC} is not in the correct order' - - def how_to_sort(self): - sorted_apps = sorted(self.apps) - unsorted_apps = self.apps.copy() - for i in range(len(sorted_apps)): - if sorted_apps[i] != unsorted_apps[i]: - # Color it - sorted_apps[i] = f'{sorted_apps[i]}' - unsorted_apps[i] = f'{unsorted_apps[i]}' - - return sorted_apps, unsorted_apps - - def __str__(self): - return str(self.apps) - - def __repr__(self): - return self.__str__() - -def main(): - # start of the Apps section - APPS_LINE_START = '## – Apps –\n' - with open('README.md', 'r') as readme_file: - lines = readme_file.readlines() - - index: int - try: - index = lines.index(APPS_LINE_START) - except ValueError: - print(f'String "{APPS_LINE_START}" was not found in README.md it\'s needed to determine the start of the app categories') - exit(1) - - categories = [] - for i in range(index+1, len(lines)): - # This is a category - if lines[i].startswith("### •"): - category = Category(lines[i][6:-1]) - categories.append(category) - # This is also a category - elif lines[i].startswith("## –"): - category_name = re.findall("(?<=##\\s–\\s).*?(?=\\s–)", lines[i])[0] - category = Category(category_name) - categories.append(category) - # This is an app - elif lines[i].startswith("*"): - if not categories: - raise RuntimeError("App entry found before any category header") - # The last category in the categories list is the one we're working on - category = categories[-1] - category.add_app(lines[i]) - - all_sorted = True - for i in categories: - if not i.is_sorted(): - print(f'Category {bcolors.BLUE}{i.name}{bcolors.ENDC} is not sorted') - print(' ' + i.where_unsorted()) - print(' Should be sorted as follows:') - sorted, unsorted = i.how_to_sort() - longest_str = len(max(unsorted, key=len)) - for j in range(len(sorted)): - color = sorted[j] != unsorted[j] - print(f' {bcolors.RED if color else ""}{unsorted[j]}{bcolors.ENDC if color else ""}{((longest_str-len(unsorted[j]))+2) * " "}{bcolors.GREEN if color else ""}{sorted[j]}{bcolors.ENDC if color else ""}') - all_sorted = False - - if not all_sorted: - exit(2) - -if __name__ == "__main__": - main()