diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index d08ca05da2a..b83123d211d 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -191,3 +191,71 @@ jobs: with: name: wheel path: python/dist/* + + # Wraps the signed binaries from the cli job into PyPI wheels (project + # "databricks"). Kept as a separate artifact from "wheel" (databricks-bundles) + # so each artifact maps to one PyPI project in the databricks-cli release workflow. + cli-wheels: + needs: cli + runs-on: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + permissions: + id-token: write + contents: read + + steps: + # The script and LICENSE come from the workflow's own ref, like the + # local actions in the other jobs, so building an older tag still works. + - name: Checkout workflow ref + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + sparse-checkout: | + .github + tools/build_cli_wheels.py + tools/validate_cli_wheels.py + tools/requirements.txt + LICENSE + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Install uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + version: "0.8.9" + + - name: Download signed CLI binaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: cli + path: cli-artifact + + - name: Build wheels + run: python3 tools/build_cli_wheels.py cli-artifact cli-wheels + + - name: Check wheel metadata + run: uvx twine@6.2.0 check --strict cli-wheels/* + + - name: Validate wheels + run: uv run -q -p ">=3.11" --no-project --with-requirements tools/requirements.txt python tools/validate_cli_wheels.py cli-wheels + + # uv picks the wheel for the runner's platform from the directory, as it would + # from PyPI. --no-index keeps it from resolving "databricks" on the index. + - name: Smoke test the wheel for this runner + run: | + cli_version=$(basename cli-artifact/databricks_cli_*_SHA256SUMS) + cli_version=${cli_version#databricks_cli_} + cli_version=${cli_version%_SHA256SUMS} + uv venv "$RUNNER_TEMP/venv" + uv pip install --python "$RUNNER_TEMP/venv" --no-index --find-links cli-wheels databricks + "$RUNNER_TEMP/venv/bin/databricks" --version | tee /dev/stderr | grep -F "$cli_version" + + - name: Upload CLI wheels + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: cli-wheels + path: cli-wheels/*.whl + if-no-files-found: error diff --git a/Taskfile.yml b/Taskfile.yml index 0c4184c9778..2ad60054377 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -265,6 +265,13 @@ tasks: cmds: - 'uv run -p ">=3.11" --no-project python tools/validate_codeowners.py' + test-cli-wheels: + # release-build.yml runs the scripts themselves on the release binaries. + desc: Unit-test the scripts that build and validate the CLI wheels for PyPI + dir: tools + cmds: + - 'uv run -q -p ">=3.11" --no-project --with-requirements requirements.txt python -m unittest -q test_build_cli_wheels test_validate_cli_wheels' + changelog-preview: desc: Print the CHANGELOG.md section the next release would add from .nextchanges/ env: @@ -314,6 +321,7 @@ tasks: - task: check-codeowners - task: check-lockfiles - task: check-agents-setup + - task: test-cli-wheels install-pythons: desc: Install Python 3.9-3.13 via uv diff --git a/acceptance/cmd/pypi-wheel/make_archives.py b/acceptance/cmd/pypi-wheel/make_archives.py new file mode 100644 index 00000000000..3d1867700d9 --- /dev/null +++ b/acceptance/cmd/pypi-wheel/make_archives.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Fake goreleaser output for the CLI wheel builder. + +Only the archive for the platform running the test holds the CLI binary. The +others hold a placeholder that cannot be executed, so running the installed +`databricks` fails if uv picks another platform's wheel. +""" + +import io +import platform +import sys +import tarfile +import zipfile +from pathlib import Path + +GOOS = {"Linux": "linux", "Darwin": "darwin", "Windows": "windows"} +GOARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"} + +cli, out = Path(sys.argv[1]), Path(sys.argv[2]) +host = (GOOS[platform.system()], GOARCH[platform.machine().lower()]) +placeholder = b"placeholder for a binary of another platform\n" + +out.mkdir() +for goos in ("linux", "darwin", "windows"): + for goarch in ("amd64", "arm64"): + data = cli.read_bytes() if (goos, goarch) == host else placeholder + if goos == "windows": + path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.zip" + with zipfile.ZipFile(path, "w") as z: + z.writestr("databricks.exe", data) + else: + path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.tar.gz" + # Compression only costs test time: the builder reads the archive either way. + with tarfile.open(path, "w:gz", compresslevel=0) as t: + info = tarfile.TarInfo("databricks") + info.size = len(data) + info.mode = 0o755 + t.addfile(info, io.BytesIO(data)) +# Only the file name is used: it carries the version. +(out / "databricks_cli_1.2.3_SHA256SUMS").touch() diff --git a/acceptance/cmd/pypi-wheel/out.test.toml b/acceptance/cmd/pypi-wheel/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/pypi-wheel/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/pypi-wheel/output.txt b/acceptance/cmd/pypi-wheel/output.txt new file mode 100644 index 00000000000..0403a0ea7a0 --- /dev/null +++ b/acceptance/cmd/pypi-wheel/output.txt @@ -0,0 +1,114 @@ + +>>> python3 make_archives.py [CLI] archives + +>>> ls archives +databricks_cli_1.2.3_SHA256SUMS +databricks_cli_1.2.3_darwin_amd64.tar.gz +databricks_cli_1.2.3_darwin_arm64.tar.gz +databricks_cli_1.2.3_linux_amd64.tar.gz +databricks_cli_1.2.3_linux_arm64.tar.gz +databricks_cli_1.2.3_windows_amd64.zip +databricks_cli_1.2.3_windows_arm64.zip + +>>> python3 [TESTROOT]/../tools/build_cli_wheels.py archives wheels [TESTROOT]/../LICENSE +wheels/databricks-1.2.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64.whl +wheels/databricks-1.2.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl +wheels/databricks-1.2.3-py3-none-macosx_13_0_x86_64.whl +wheels/databricks-1.2.3-py3-none-macosx_13_0_arm64.whl +wheels/databricks-1.2.3-py3-none-win_amd64.whl +wheels/databricks-1.2.3-py3-none-win_arm64.whl + +>>> uv run -q --no-project --with packaging python [TESTROOT]/../tools/validate_cli_wheels.py wheels +valid: wheels/databricks-1.2.3-py3-none-macosx_13_0_arm64.whl +valid: wheels/databricks-1.2.3-py3-none-macosx_13_0_x86_64.whl +valid: wheels/databricks-1.2.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl +valid: wheels/databricks-1.2.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64.whl +valid: wheels/databricks-1.2.3-py3-none-win_amd64.whl +valid: wheels/databricks-1.2.3-py3-none-win_arm64.whl + +>>> python3 show_wheels.py wheels +== databricks-1.2.3-py3-none-macosx_13_0_arm64.whl +100755 databricks-1.2.3.data/scripts/databricks +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-macosx_13_0_arm64 +== databricks-1.2.3-py3-none-macosx_13_0_x86_64.whl +100755 databricks-1.2.3.data/scripts/databricks +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-macosx_13_0_x86_64 +== databricks-1.2.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl +100755 databricks-1.2.3.data/scripts/databricks +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-manylinux_2_17_aarch64 +Tag: py3-none-manylinux2014_aarch64 +Tag: py3-none-musllinux_1_1_aarch64 +== databricks-1.2.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64.whl +100755 databricks-1.2.3.data/scripts/databricks +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-manylinux_2_17_x86_64 +Tag: py3-none-manylinux2014_x86_64 +Tag: py3-none-musllinux_1_1_x86_64 +== databricks-1.2.3-py3-none-win_amd64.whl +100755 databricks-1.2.3.data/scripts/databricks.exe +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-win_amd64 +== databricks-1.2.3-py3-none-win_arm64.whl +100755 databricks-1.2.3.data/scripts/databricks.exe +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-win_arm64 +== METADATA (same in every wheel) +Metadata-Version: 2.4 +Name: databricks +Version: 1.2.3 +Summary: Databricks CLI +Project-URL: Source, https://github.com/databricks/cli +Project-URL: Documentation, https://docs.databricks.com/dev-tools/cli/ +License-File: LICENSE +Description-Content-Type: text/markdown + +# Databricks CLI + +Command-line interface for Databricks. This package installs the prebuilt `databricks` binary. + +Run it with `uvx databricks` or `pipx run databricks`, or install it with `uv tool install databricks`. + +Documentation: https://docs.databricks.com/dev-tools/cli/ +Source: https://github.com/databricks/cli + +>>> uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version +Databricks CLI v[CLI_VERSION] diff --git a/acceptance/cmd/pypi-wheel/script b/acceptance/cmd/pypi-wheel/script new file mode 100644 index 00000000000..df4ad509a0d --- /dev/null +++ b/acceptance/cmd/pypi-wheel/script @@ -0,0 +1,15 @@ +# Package the CLI under test the way release-build does, then run it through uvx +# to check that a wheel built by build_cli_wheels.py is usable by Python tooling. +trace python3 make_archives.py $CLI archives +trace ls archives +trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE +# Offline, so packaging comes from the vendored packages rather than the +# tools/requirements.txt pin (they can differ after a Dependabot bump). +trace uv run -q --no-project --with packaging python $TESTROOT/../tools/validate_cli_wheels.py wheels +trace python3 show_wheels.py wheels + +# uv must pick the wheel for this platform: the other wheels hold a placeholder +# instead of the binary (see make_archives.py), so a wrong pick fails here. +trace uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version + +rm -fr archives wheels diff --git a/acceptance/cmd/pypi-wheel/show_wheels.py b/acceptance/cmd/pypi-wheel/show_wheels.py new file mode 100644 index 00000000000..53ba6090b4a --- /dev/null +++ b/acceptance/cmd/pypi-wheel/show_wheels.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Print what each wheel in a directory installs, in a platform-independent form. + +RECORD is not printed: its hashes cover the CLI binary, which differs per build. +""" + +import sys +import zipfile +from pathlib import Path + +wheels = sorted(Path(sys.argv[1]).glob("*.whl")) +for whl in wheels: + print(f"== {whl.name}") + with zipfile.ZipFile(whl) as z: + for info in z.infolist(): + print(f"{info.external_attr >> 16:o} {info.filename}") + wheel_file = next(n for n in z.namelist() if n.endswith(".dist-info/WHEEL")) + print(z.read(wheel_file).decode(), end="") + +with zipfile.ZipFile(wheels[0]) as z: + metadata_file = next(n for n in z.namelist() if n.endswith(".dist-info/METADATA")) + print("== METADATA (same in every wheel)") + print(z.read(metadata_file).decode(), end="") diff --git a/acceptance/cmd/pypi-wheel/test.toml b/acceptance/cmd/pypi-wheel/test.toml new file mode 100644 index 00000000000..23eaa49b0c6 --- /dev/null +++ b/acceptance/cmd/pypi-wheel/test.toml @@ -0,0 +1,5 @@ +Cloud = false + +# uvx may need to create a Python environment. +Timeout = '180s' +TimeoutWindows = '240s' diff --git a/tools/build_cli_wheels.py b/tools/build_cli_wheels.py new file mode 100644 index 00000000000..8bff20792ca --- /dev/null +++ b/tools/build_cli_wheels.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Package the released Databricks CLI binaries as PyPI wheels. + +Input is the directory of the `cli` release-build artifact: the goreleaser +archives (databricks_cli_*__.{tar.gz,zip}) and the SHA256SUMS file, whose name carries the version. +The binaries are taken from the archives as-is, so the signed Windows binaries +are what ends up in the wheels. + +Each wheel is py3-none-: platform-specific but independent of the +Python version. It contains only the binary, installed to the environment's +scripts directory (bin/ or Scripts/) via the wheel `.data/scripts` mechanism, +and no Python package, so `databricks` and `uvx databricks` run the binary +directly without a Python bootstrapper. + +Usage: build_cli_wheels.py [license-file] +""" + +import base64 +import hashlib +import io +import re +import sys +import tarfile +import zipfile +from pathlib import Path + +DIST_NAME = "databricks" +SUMMARY = "Databricks CLI" +DESCRIPTION = """# Databricks CLI + +Command-line interface for Databricks. This package installs the prebuilt `databricks` binary. + +Run it with `uvx databricks` or `pipx run databricks`, or install it with `uv tool install databricks`. + +Documentation: https://docs.databricks.com/dev-tools/cli/ +Source: https://github.com/databricks/cli +""" + +# (goreleaser os, goreleaser arch) -> wheel platform tag. Compressed tag sets +# (dot-separated) make one wheel serve glibc and musl Linux: the binary is +# statically linked (CGO_ENABLED=0). +PLATFORMS = { + ("linux", "amd64"): "manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64", + ("linux", "arm64"): "manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64", + # Go 1.27 binaries require macOS 13 (LC_BUILD_VERSION minos); a lower tag + # would install a binary that cannot start on older macOS. + ("darwin", "amd64"): "macosx_13_0_x86_64", + ("darwin", "arm64"): "macosx_13_0_arm64", + ("windows", "amd64"): "win_amd64", + ("windows", "arm64"): "win_arm64", +} + +# Fixed timestamp so rebuilding from the same artifact yields identical wheels. +ZIP_DATE_TIME = (1980, 1, 1, 0, 0, 0) + + +def posix_list(paths: list[Path]) -> list[str]: + """Paths for messages, with forward slashes so output is the same on Windows.""" + return sorted(p.as_posix() for p in paths) + + +def read_version(artifact_dir: Path) -> str: + """Return the CLI version from the databricks_cli__SHA256SUMS file name.""" + sums_files = list(artifact_dir.glob("databricks_cli_*_SHA256SUMS")) + if len(sums_files) != 1: + raise SystemExit( + f"expected exactly one databricks_cli_*_SHA256SUMS in {artifact_dir.as_posix()}, found {posix_list(sums_files)}" + ) + return sums_files[0].name[len("databricks_cli_") : -len("_SHA256SUMS")] + + +def pep440(cli_version: str) -> str: + """Convert a goreleaser version to PEP 440. + + Releases (1.23.0) are unchanged. Snapshots (1.23.1-dev+abc123) become + 1.23.1.dev0+abc123. + """ + m = re.fullmatch(r"(\d+\.\d+\.\d+)(?:-dev(?:\+([0-9A-Za-z.]+))?)?", cli_version) + if not m: + raise SystemExit(f"unsupported CLI version {cli_version!r}") + base, local = m.groups() + if cli_version == base: + return base + return f"{base}.dev0" + (f"+{local}" if local else "") + + +def find_archive(artifact_dir: Path, goos: str, goarch: str) -> Path: + ext = "zip" if goos == "windows" else "tar.gz" + matches = list(artifact_dir.glob(f"databricks_cli_*{goos}_{goarch}.{ext}")) + if len(matches) != 1: + raise SystemExit( + f"expected exactly one {goos}/{goarch} .{ext} archive in {artifact_dir.as_posix()}, found {posix_list(matches)}" + ) + return matches[0] + + +def extract_binary(archive: Path, binary_name: str) -> bytes: + if archive.name.endswith(".zip"): + with zipfile.ZipFile(archive) as z: + return z.read(binary_name) + with tarfile.open(archive) as t: + f = t.extractfile(binary_name) + if f is None: + raise SystemExit(f"{binary_name} is not a regular file in {archive.as_posix()}") + return f.read() + + +def record_hash(data: bytes) -> str: + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + return f"sha256={digest},{len(data)}" + + +def build_wheel( + out_dir: Path, + version: str, + platform_tag: str, + binary_name: str, + binary: bytes, + license_text: bytes, +) -> Path: + dist_info = f"{DIST_NAME}-{version}.dist-info" + data_dir = f"{DIST_NAME}-{version}.data" + + metadata = ( + # 2.4 is the first version that allows License-File; PyPI validates this. + "Metadata-Version: 2.4\n" + f"Name: {DIST_NAME}\n" + f"Version: {version}\n" + f"Summary: {SUMMARY}\n" + "Project-URL: Source, https://github.com/databricks/cli\n" + "Project-URL: Documentation, https://docs.databricks.com/dev-tools/cli/\n" + "License-File: LICENSE\n" + "Description-Content-Type: text/markdown\n" + f"\n{DESCRIPTION}" + ).encode() + wheel_meta = ( + "Wheel-Version: 1.0\n" + "Generator: databricks-cli build_cli_wheels.py\n" + "Root-Is-Purelib: false\n" + "".join(f"Tag: py3-none-{t}\n" for t in platform_tag.split(".")) + ).encode() + + # name -> (content, unix mode) + files = { + f"{data_dir}/scripts/{binary_name}": (binary, 0o755), + f"{dist_info}/METADATA": (metadata, 0o644), + f"{dist_info}/WHEEL": (wheel_meta, 0o644), + f"{dist_info}/licenses/LICENSE": (license_text, 0o644), + } + record = "".join(f"{name},{record_hash(data)}\n" for name, (data, _) in files.items()) + record += f"{dist_info}/RECORD,,\n" + files[f"{dist_info}/RECORD"] = (record.encode(), 0o644) + + out = out_dir / f"{DIST_NAME}-{version}-py3-none-{platform_tag}.whl" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: + for name, (data, mode) in files.items(): + info = zipfile.ZipInfo(name, ZIP_DATE_TIME) + info.external_attr = (0o100000 | mode) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, data) + out.write_bytes(buf.getvalue()) + return out + + +def main() -> None: + if len(sys.argv) not in (3, 4): + raise SystemExit(__doc__) + artifact_dir = Path(sys.argv[1]) + out_dir = Path(sys.argv[2]) + license_file = Path(sys.argv[3]) if len(sys.argv) == 4 else Path(__file__).resolve().parents[1] / "LICENSE" + + cli_version = read_version(artifact_dir) + version = pep440(cli_version) + license_text = license_file.read_bytes() + out_dir.mkdir(parents=True, exist_ok=True) + + for (goos, goarch), platform_tag in PLATFORMS.items(): + archive = find_archive(artifact_dir, goos, goarch) + binary_name = "databricks.exe" if goos == "windows" else "databricks" + binary = extract_binary(archive, binary_name) + print(build_wheel(out_dir, version, platform_tag, binary_name, binary, license_text).as_posix()) + + +if __name__ == "__main__": + main() diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 00000000000..f136ca4e458 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,3 @@ +# Python packages for the scripts in tools/ that are not stdlib-only. Pinned here, +# not inline in workflows or the Taskfile, so Dependabot tracks them. +packaging==25.0 diff --git a/tools/test_build_cli_wheels.py b/tools/test_build_cli_wheels.py new file mode 100644 index 00000000000..b7d75d63748 --- /dev/null +++ b/tools/test_build_cli_wheels.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +import io +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path + +import build_cli_wheels as b + + +class TestBuildCliWheels(unittest.TestCase): + def test_pep440(self): + self.assertEqual(b.pep440("1.23.0"), "1.23.0") + self.assertEqual(b.pep440("1.23.1-dev+abc123"), "1.23.1.dev0+abc123") + with self.assertRaises(SystemExit): + b.pep440("v1.23.0") + + def test_end_to_end(self): + with tempfile.TemporaryDirectory() as tmp: + art, out = Path(tmp, "art"), Path(tmp, "out") + art.mkdir() + for (goos, goarch), _ in b.PLATFORMS.items(): + if goos == "windows": + path = art / f"databricks_cli_1.23.0_{goos}_{goarch}.zip" + with zipfile.ZipFile(path, "w") as z: + z.writestr("databricks.exe", b"MZ") + else: + path = art / f"databricks_cli_1.23.0_{goos}_{goarch}.tar.gz" + with tarfile.open(path, "w:gz") as t: + info = tarfile.TarInfo("databricks") + info.size = 3 + t.addfile(info, io.BytesIO(b"ELF")) + (art / "databricks_cli_1.23.0_SHA256SUMS").touch() + lic = Path(tmp, "LICENSE") + lic.write_text("license") + + b.sys.argv = ["x", str(art), str(out), str(lic)] + b.main() + + wheels = sorted(p.name for p in out.glob("*.whl")) + self.assertEqual(len(wheels), len(b.PLATFORMS)) + self.assertIn("databricks-1.23.0-py3-none-win_amd64.whl", wheels) + with zipfile.ZipFile(out / "databricks-1.23.0-py3-none-macosx_13_0_arm64.whl") as z: + script = z.getinfo("databricks-1.23.0.data/scripts/databricks") + self.assertEqual(script.external_attr >> 16, 0o100755) + self.assertIn("Tag: py3-none-macosx_13_0_arm64", z.read("databricks-1.23.0.dist-info/WHEEL").decode()) + metadata = z.read("databricks-1.23.0.dist-info/METADATA").decode() + self.assertTrue(metadata.startswith("Metadata-Version: 2.4\n")) + self.assertIn("License-File: LICENSE\n", metadata) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test_validate_cli_wheels.py b/tools/test_validate_cli_wheels.py new file mode 100644 index 00000000000..baf183468c2 --- /dev/null +++ b/tools/test_validate_cli_wheels.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import tempfile +import unittest +import zipfile +from pathlib import Path + +import build_cli_wheels as b +import validate_cli_wheels as v + +VERSION = "1.23.0" +LINUX_WHEEL = f"databricks-{VERSION}-py3-none-{b.PLATFORMS[('linux', 'amd64')]}.whl" +DIST_INFO = f"databricks-{VERSION}.dist-info" +SCRIPT = f"databricks-{VERSION}.data/scripts/databricks" + + +def build_wheels(out: Path) -> None: + out.mkdir() + for (goos, _), tag in b.PLATFORMS.items(): + binary_name = "databricks.exe" if goos == "windows" else "databricks" + b.build_wheel(out, VERSION, tag, binary_name, b"binary", b"license") + + +def rewrite(whl: Path, changes: dict, fix_record: bool = True, new_name: str = "") -> None: + """Rewrite a wheel with files replaced (bytes) or removed (None), optionally keeping RECORD consistent.""" + with zipfile.ZipFile(whl) as z: + files = {i.filename: (i, z.read(i)) for i in z.infolist()} + for name, data in changes.items(): + if data is None: + del files[name] + else: + info = files[name][0] if name in files else zipfile.ZipInfo(name, b.ZIP_DATE_TIME) + files[name] = (info, data) + if fix_record: + record_name = f"{DIST_INFO}/RECORD" + record = "".join(f"{n},{b.record_hash(d)}\n" for n, (_, d) in files.items() if n != record_name) + files[record_name] = (files[record_name][0], (record + f"{record_name},,\n").encode()) + whl.unlink() + with zipfile.ZipFile(whl.with_name(new_name or whl.name), "w", zipfile.ZIP_DEFLATED) as z: + for info, data in files.values(): + z.writestr(info, data) + + +class TestValidateCliWheels(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.out = Path(tmp.name, "wheels") + build_wheels(self.out) + self.linux = self.out / LINUX_WHEEL + + def assert_invalid(self, message: str): + with self.assertRaises(SystemExit) as cm: + v.validate_dir(self.out) + self.assertIn(message, str(cm.exception)) + + def test_valid(self): + self.assertEqual(len(v.validate_dir(self.out)), 6) + + def test_metadata_version_too_old_for_license_file(self): + with zipfile.ZipFile(self.linux) as z: + metadata = z.read(f"{DIST_INFO}/METADATA").replace(b"Metadata-Version: 2.4", b"Metadata-Version: 2.1") + rewrite(self.linux, {f"{DIST_INFO}/METADATA": metadata}) + self.assert_invalid("license-file introduced in metadata version 2.4") + + def test_metadata_version_mismatch(self): + with zipfile.ZipFile(self.linux) as z: + metadata = z.read(f"{DIST_INFO}/METADATA").replace(b"Version: 1.23.0", b"Version: 1.23.1") + rewrite(self.linux, {f"{DIST_INFO}/METADATA": metadata}) + self.assert_invalid("do not match the filename") + + def test_record_hash_mismatch(self): + rewrite(self.linux, {SCRIPT: b"tampered"}, fix_record=False) + self.assert_invalid(f"RECORD: wrong hash or size for {SCRIPT}") + + def test_extra_file(self): + rewrite(self.linux, {"evil.pth": b"import os"}) + self.assert_invalid("unexpected files: ['evil.pth']") + + def test_missing_license(self): + rewrite(self.linux, {f"{DIST_INFO}/licenses/LICENSE": None}) + self.assert_invalid("unexpected files") + + def test_wheel_tags_differ_from_filename(self): + rewrite(self.linux, {}, new_name=f"databricks-{VERSION}-py3-none-manylinux_2_28_x86_64.whl") + self.assert_invalid("do not match the filename") + + def test_missing_platform(self): + next(self.out.glob("*win_arm64.whl")).unlink() + self.assert_invalid("missing wheels for [('windows', 'arm64')]") + + def test_error_paths_use_forward_slashes(self): + rewrite(self.linux, {"evil.pth": b"import os"}) + self.assert_invalid(f"{self.linux.as_posix()}: ") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate_cli_wheels.py b/tools/validate_cli_wheels.py new file mode 100644 index 00000000000..a3b49d699b7 --- /dev/null +++ b/tools/validate_cli_wheels.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Validate the Databricks CLI wheels built by build_cli_wheels.py. + +Checks what a build backend would otherwise guarantee and what pip, uv and +PyPI rely on: the filename, METADATA (validated by `packaging`, which is +stricter than `twine check`), WHEEL tags, RECORD hashes and sizes, and that +each wheel installs only the CLI binary. Also requires exactly one wheel per +supported platform, all with the same version. + +Requires `packaging` (see requirements.txt). Usage: validate_cli_wheels.py +""" + +import base64 +import csv +import email.parser +import hashlib +import io +import sys +import zipfile +from pathlib import Path + +from packaging.metadata import Metadata +from packaging.utils import parse_wheel_filename + +DIST_NAME = "databricks" +EXPECTED_PLATFORMS = {(goos, goarch) for goos in ("linux", "darwin", "windows") for goarch in ("amd64", "arm64")} +OS_PREFIXES = (("manylinux", "linux"), ("musllinux", "linux"), ("macosx", "darwin"), ("win", "windows")) +ARCH_SUFFIXES = (("x86_64", "amd64"), ("amd64", "amd64"), ("aarch64", "arm64"), ("arm64", "arm64")) + + +class InvalidWheel(Exception): + pass + + +def platform_of(tag: str) -> tuple[str, str]: + goos = next((o for prefix, o in OS_PREFIXES if tag.startswith(prefix)), None) + goarch = next((a for suffix, a in ARCH_SUFFIXES if tag.endswith(suffix)), None) + if goos is None or goarch is None: + raise InvalidWheel(f"unsupported platform tag {tag}") + return goos, goarch + + +def record_hash(data: bytes) -> str: + return "sha256=" + base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + + +def check_record(z: zipfile.ZipFile, record_name: str) -> None: + rows = list(csv.reader(io.StringIO(z.read(record_name).decode()))) + listed = {} + for row in rows: + if len(row) != 3: + raise InvalidWheel(f"RECORD: malformed row {row}") + if row[0] in listed: + raise InvalidWheel(f"RECORD: {row[0]} listed twice") + listed[row[0]] = row + names = z.namelist() + if len(names) != len(set(names)): + raise InvalidWheel("duplicate file in the zip") + if set(listed) != set(names): + raise InvalidWheel(f"RECORD does not match the zip: {sorted(set(listed) ^ set(names))}") + for name, (_, digest, size) in listed.items(): + if name == record_name: + if digest or size: + raise InvalidWheel("RECORD must list itself without a hash or size") + continue + data = z.read(name) + if digest != record_hash(data) or size != str(len(data)): + raise InvalidWheel(f"RECORD: wrong hash or size for {name}") + + +def validate_wheel(path: Path) -> tuple[str, tuple[str, str]]: + """Validate one wheel; return its version and (goos, goarch).""" + name, version, build, tags = parse_wheel_filename(path.name) + if name != DIST_NAME or build: + raise InvalidWheel(f"unexpected name or build tag in {path.name}") + if {(t.interpreter, t.abi) for t in tags} != {("py3", "none")}: + raise InvalidWheel(f"expected py3-none tags, got {sorted(map(str, tags))}") + platforms = {platform_of(t.platform) for t in tags} + if len(platforms) != 1: + raise InvalidWheel(f"tags cover more than one platform: {sorted(platforms)}") + platform = platforms.pop() + + dist_info = f"{DIST_NAME}-{version}.dist-info" + script = f"{DIST_NAME}-{version}.data/scripts/" + ("databricks.exe" if platform[0] == "windows" else "databricks") + expected_files = { + script, + f"{dist_info}/METADATA", + f"{dist_info}/WHEEL", + f"{dist_info}/RECORD", + f"{dist_info}/licenses/LICENSE", + } + + with zipfile.ZipFile(path) as z: + if set(z.namelist()) != expected_files: + raise InvalidWheel(f"unexpected files: {sorted(set(z.namelist()) ^ expected_files)}") + if z.getinfo(script).external_attr >> 16 != 0o100755: + raise InvalidWheel(f"{script} is not a regular file with mode 755") + check_record(z, f"{dist_info}/RECORD") + + try: + metadata = Metadata.from_email(z.read(f"{dist_info}/METADATA"), validate=True) + except Exception as e: + details = "; ".join(str(x) for x in getattr(e, "exceptions", [e])) + raise InvalidWheel(f"METADATA: {details}") from None + if metadata.name != DIST_NAME or metadata.version != version: + raise InvalidWheel(f"METADATA name/version {metadata.name} {metadata.version} do not match the filename") + if metadata.license_files != ["LICENSE"]: + raise InvalidWheel(f"METADATA: expected License-File: LICENSE, got {metadata.license_files}") + + wheel = email.parser.BytesParser().parsebytes(z.read(f"{dist_info}/WHEEL")) + if wheel["Wheel-Version"] != "1.0" or wheel["Root-Is-Purelib"] != "false": + raise InvalidWheel("WHEEL: expected Wheel-Version 1.0 and Root-Is-Purelib false") + if set(wheel.get_all("Tag", [])) != {str(t) for t in tags}: + raise InvalidWheel(f"WHEEL tags {wheel.get_all('Tag')} do not match the filename") + + return str(version), platform + + +def validate_dir(wheels_dir: Path) -> list[Path]: + wheels = sorted(wheels_dir.glob("*.whl")) + seen = {} + versions = set() + for whl in wheels: + try: + version, platform = validate_wheel(whl) + except (InvalidWheel, ValueError) as e: + raise SystemExit(f"{whl.as_posix()}: {e}") from None + if platform in seen: + raise SystemExit(f"{whl.as_posix()}: second wheel for {platform}, after {seen[platform].as_posix()}") + seen[platform] = whl + versions.add(version) + if set(seen) != EXPECTED_PLATFORMS: + raise SystemExit(f"{wheels_dir.as_posix()}: missing wheels for {sorted(EXPECTED_PLATFORMS - set(seen))}") + if len(versions) != 1: + raise SystemExit(f"{wheels_dir.as_posix()}: wheels have different versions {sorted(versions)}") + return wheels + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit(__doc__) + for whl in validate_dir(Path(sys.argv[1])): + print(f"valid: {whl.as_posix()}") + + +if __name__ == "__main__": + main()