Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions .github/scripts/check_upstream_majors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""Find frameworks whose upstream shipped a stable major the store has no definition for.

Each framework's composer detect packages are looked up on Packagist, in order,
and the newest stable major is compared with the index's `latest`. Prints the
findings as JSON for the workflow to turn into issues, and lists what it could
not check in the job summary.

python3 .github/scripts/check_upstream_majors.py
"""

import json
import os
import pathlib
import urllib.error
import urllib.request

ROOT = pathlib.Path(__file__).resolve().parents[2]

# Frameworks detected by files alone still ship on Packagist under a name the
# detect rules never needed.
EXTRA_PACKAGES = {
"bedrock": ["roots/bedrock"],
"wordpress": ["roots/wordpress"],
"testbench": ["orchestra/testbench"],
}


def fetch_releases(package):
url = f"https://repo.packagist.org/p2/{package}.json"
req = urllib.request.Request(url, headers={"User-Agent": "lerd-frameworks-upstream-check"})
with urllib.request.urlopen(req, timeout=30) as resp:
return json.load(resp)["packages"][package]


def newest_stable_major(releases):
majors = [int(r["version_normalized"].split(".")[0])
for r in releases if "-" not in r["version_normalized"]]
return max(majors, default=None)


def packages_for(framework):
detected = [d["composer"] for d in framework["detect"] if "composer" in d]
return detected + EXTRA_PACKAGES.get(framework["name"], [])


def upstream_major(packages, fetch):
# A 404 means the package lives off Packagist (Magento's own repo), so the
# next candidate is tried; anything else is a real failure.
for package in packages:
try:
return package, newest_stable_major(fetch(package))
except urllib.error.HTTPError as err:
if err.code != 404:
raise
return None, None


def find_new_majors(index, fetch):
found, unchecked = [], []
for fw in index["frameworks"]:
package, upstream = upstream_major(packages_for(fw), fetch)
if upstream is None:
unchecked.append(fw["name"])
continue
latest = int(fw["latest"])
if upstream > latest:
found.append({"name": fw["name"], "label": fw["label"], "package": package,
"latest": latest, "upstream": upstream})
return found, unchecked


def main():
index = json.loads((ROOT / "frameworks" / "index.json").read_text())
found, unchecked = find_new_majors(index, fetch_releases)
summary = os.environ.get("GITHUB_STEP_SUMMARY")
if summary and unchecked:
with open(summary, "a") as out:
out.write("Not on Packagist, check by hand: " + ", ".join(unchecked) + "\n")
print(json.dumps(found))


if __name__ == "__main__":
main()
59 changes: 59 additions & 0 deletions .github/scripts/test_check_upstream_majors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
import unittest
import urllib.error

from check_upstream_majors import find_new_majors, newest_stable_major


def releases(*versions):
return [{"version_normalized": v} for v in versions]


class NewestStableMajorTest(unittest.TestCase):
def test_ignores_pre_releases(self):
self.assertEqual(newest_stable_major(releases("14.0.0.0-beta1", "13.2.0.0")), 13)

def test_none_without_a_stable_release(self):
self.assertIsNone(newest_stable_major(releases("1.0.0.0-RC1")))


class FindNewMajorsTest(unittest.TestCase):
def index(self, latest="13", detect=None):
return {"frameworks": [{"name": "laravel", "label": "Laravel", "latest": latest,
"detect": detect or [{"composer": "laravel/framework"}]}]}

def test_reports_a_major_above_latest(self):
found, unchecked = find_new_majors(self.index(), lambda pkg: releases("14.0.0.0", "13.9.0.0"))
self.assertEqual(found, [{"name": "laravel", "label": "Laravel", "package": "laravel/framework",
"latest": 13, "upstream": 14}])
self.assertEqual(unchecked, [])

def test_quiet_when_store_is_current(self):
found, _ = find_new_majors(self.index(), lambda pkg: releases("13.9.0.0"))
self.assertEqual(found, [])

def test_falls_through_to_the_next_package_on_404(self):
def fetch(pkg):
if pkg == "a/missing":
raise urllib.error.HTTPError(pkg, 404, "Not Found", None, None)
return releases("14.0.0.0")

index = self.index(detect=[{"composer": "a/missing"}, {"composer": "laravel/framework"}])
found, _ = find_new_majors(index, fetch)
self.assertEqual(found[0]["package"], "laravel/framework")

def test_lists_frameworks_it_cannot_check(self):
index = self.index(detect=[{"file": "artisan"}])
index["frameworks"][0]["name"] = "nocomposer"
found, unchecked = find_new_majors(index, lambda pkg: self.fail("fetched " + pkg))
self.assertEqual((found, unchecked), ([], ["nocomposer"]))

def test_other_http_errors_fail_loudly(self):
def fetch(pkg):
raise urllib.error.HTTPError(pkg, 500, "Server Error", None, None)

with self.assertRaises(urllib.error.HTTPError):
find_new_majors(self.index(), fetch)


if __name__ == "__main__":
unittest.main()
49 changes: 49 additions & 0 deletions .github/workflows/upstream-majors.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
name: Upstream majors

on:
schedule:
- cron: "0 7 * * 1"
workflow_dispatch:

# Opens one issue per framework major that upstream released and the store has
# no definition for yet. An issue with the same title, open or closed, is never
# opened twice.
jobs:
check:
runs-on: ubuntu-latest
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@v4

- uses: actions/setup-python@v5
with:
python-version: "3.12"

- name: Test the checker
run: python3 -m unittest discover -s .github/scripts -p "test_*.py"

- name: Find new majors
id: find
run: echo "found=$(python3 .github/scripts/check_upstream_majors.py)" >> "$GITHUB_OUTPUT"

- name: Open issues
env:
GH_TOKEN: ${{ github.token }}
FOUND: ${{ steps.find.outputs.found }}
run: |
set -euo pipefail
echo "$FOUND" | jq -c '.[]' | while read -r fw; do
name=$(jq -r .name <<<"$fw")
label=$(jq -r .label <<<"$fw")
package=$(jq -r .package <<<"$fw")
latest=$(jq -r .latest <<<"$fw")
upstream=$(jq -r .upstream <<<"$fw")
title="$label $upstream is out"
if gh issue list --state all --search "\"$title\" in:title" --json title --jq '.[].title' | grep -qxF "$title"; then
echo "Already tracked: $title"
continue
fi
gh issue create --title "$title" --body "$package released a stable $upstream.x on Packagist, and the newest definition here is $latest. Add frameworks/$name/$upstream.yaml by copying $latest.yaml and checking its PHP range, workers, commands and env against the new release, then list it in frameworks/index.json. Packages under packages/ with a max for $name may need raising too."
done
Loading