From bcf6dd5e1c9ca365dccfda43b4610c26bc0048ac Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 23 Oct 2026 12:52:09 +0200 Subject: [PATCH 01/10] Build PyPI wheels for the CLI in release-build Wrap the signed release binaries into py3-none- wheels for the `databricks` PyPI project and upload them as the `cli-wheels` artifact, separate from the `wheel` (databricks-bundles) artifact. Co-authored-by: Isaac --- .github/scripts/build_cli_wheels.py | 181 +++++++++++++++++++++++ .github/scripts/test_build_cli_wheels.py | 67 +++++++++ .github/workflows/release-build.yml | 60 ++++++++ Taskfile.yml | 7 + 4 files changed, 315 insertions(+) create mode 100644 .github/scripts/build_cli_wheels.py create mode 100644 .github/scripts/test_build_cli_wheels.py diff --git a/.github/scripts/build_cli_wheels.py b/.github/scripts/build_cli_wheels.py new file mode 100644 index 00000000000..6521650d442 --- /dev/null +++ b/.github/scripts/build_cli_wheels.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Package the released Databricks CLI binaries as PyPI wheels. + +Input is the directory of the `cli` release-build artifact: the goreleaser +archives (databricks_cli_*__.{tar.gz,zip}) and the SHA256SUMS file. +The binaries are taken from the archives as-is, so the signed Windows binaries +are what ends up in the wheels. + +Each wheel is py3-none-: platform-specific but independent of the +Python version. It contains only the binary, installed to the environment's +scripts directory (bin/ or Scripts/) via the wheel `.data/scripts` mechanism, +and no Python package, so `databricks` and `uvx databricks` run the binary +directly without a Python bootstrapper. + +Usage: build_cli_wheels.py [license-file] +""" + +import base64 +import hashlib +import io +import re +import sys +import tarfile +import zipfile +from pathlib import Path + +DIST_NAME = "databricks" +SUMMARY = "Databricks CLI" +DESCRIPTION = """# Databricks CLI + +Command-line interface for Databricks. This package installs the prebuilt `databricks` binary. + +Run it with `uvx databricks` or `pipx run databricks`, or install it with `uv tool install databricks`. + +Documentation: https://docs.databricks.com/dev-tools/cli/ +Source: https://github.com/databricks/cli +""" + +# (goreleaser os, goreleaser arch) -> wheel platform tag. Compressed tag sets +# (dot-separated) make one wheel serve glibc and musl Linux: the binary is +# statically linked (CGO_ENABLED=0). +PLATFORMS = { + ("linux", "amd64"): "manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64", + ("linux", "arm64"): "manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64", + ("darwin", "amd64"): "macosx_11_0_x86_64", + ("darwin", "arm64"): "macosx_11_0_arm64", + ("windows", "amd64"): "win_amd64", + ("windows", "arm64"): "win_arm64", +} + +# Fixed timestamp so rebuilding from the same artifact yields identical wheels. +ZIP_DATE_TIME = (1980, 1, 1, 0, 0, 0) + + +def read_checksums(artifact_dir: Path) -> tuple[str, dict[str, str]]: + """Return (cli_version, {file name: sha256}) from the SHA256SUMS file.""" + sums_files = list(artifact_dir.glob("databricks_cli_*_SHA256SUMS")) + if len(sums_files) != 1: + raise SystemExit(f"expected exactly one databricks_cli_*_SHA256SUMS in {artifact_dir}, found {sums_files}") + version = sums_files[0].name[len("databricks_cli_") : -len("_SHA256SUMS")] + sums = {} + for line in sums_files[0].read_text().splitlines(): + digest, _, name = line.partition(" ") + sums[name.strip()] = digest + return version, sums + + +def pep440(cli_version: str) -> str: + """Convert a goreleaser version to PEP 440. + + Releases (1.23.0) are unchanged. Snapshots (1.23.1-dev+abc123) become + 1.23.1.dev0+abc123. + """ + m = re.fullmatch(r"(\d+\.\d+\.\d+)(?:-dev(?:\+([0-9A-Za-z.]+))?)?", cli_version) + if not m: + raise SystemExit(f"unsupported CLI version {cli_version!r}") + base, local = m.groups() + if cli_version == base: + return base + return f"{base}.dev0" + (f"+{local}" if local else "") + + +def find_archive(artifact_dir: Path, goos: str, goarch: str) -> Path: + ext = "zip" if goos == "windows" else "tar.gz" + matches = list(artifact_dir.glob(f"databricks_cli_*{goos}_{goarch}.{ext}")) + if len(matches) != 1: + raise SystemExit(f"expected exactly one {goos}/{goarch} .{ext} archive in {artifact_dir}, found {matches}") + return matches[0] + + +def extract_binary(archive: Path, binary_name: str) -> bytes: + if archive.name.endswith(".zip"): + with zipfile.ZipFile(archive) as z: + return z.read(binary_name) + with tarfile.open(archive) as t: + f = t.extractfile(binary_name) + if f is None: + raise SystemExit(f"{binary_name} is not a regular file in {archive}") + return f.read() + + +def record_hash(data: bytes) -> str: + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + return f"sha256={digest},{len(data)}" + + +def build_wheel( + out_dir: Path, + version: str, + platform_tag: str, + binary_name: str, + binary: bytes, + license_text: bytes, +) -> Path: + dist_info = f"{DIST_NAME}-{version}.dist-info" + data_dir = f"{DIST_NAME}-{version}.data" + + metadata = ( + "Metadata-Version: 2.1\n" + f"Name: {DIST_NAME}\n" + f"Version: {version}\n" + f"Summary: {SUMMARY}\n" + "Home-page: https://github.com/databricks/cli\n" + "License: Databricks License\n" + "License-File: LICENSE\n" + "Description-Content-Type: text/markdown\n" + f"\n{DESCRIPTION}" + ).encode() + wheel_meta = ( + "Wheel-Version: 1.0\n" + "Generator: databricks-cli build_cli_wheels.py\n" + "Root-Is-Purelib: false\n" + "".join(f"Tag: py3-none-{t}\n" for t in platform_tag.split(".")) + ).encode() + + # name -> (content, unix mode) + files = { + f"{data_dir}/scripts/{binary_name}": (binary, 0o755), + f"{dist_info}/METADATA": (metadata, 0o644), + f"{dist_info}/WHEEL": (wheel_meta, 0o644), + f"{dist_info}/licenses/LICENSE": (license_text, 0o644), + } + record = "".join(f"{name},{record_hash(data)}\n" for name, (data, _) in files.items()) + record += f"{dist_info}/RECORD,,\n" + files[f"{dist_info}/RECORD"] = (record.encode(), 0o644) + + out = out_dir / f"{DIST_NAME}-{version}-py3-none-{platform_tag}.whl" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z: + for name, (data, mode) in files.items(): + info = zipfile.ZipInfo(name, ZIP_DATE_TIME) + info.external_attr = (0o100000 | mode) << 16 + info.compress_type = zipfile.ZIP_DEFLATED + z.writestr(info, data) + out.write_bytes(buf.getvalue()) + return out + + +def main() -> None: + if len(sys.argv) not in (3, 4): + raise SystemExit(__doc__) + artifact_dir = Path(sys.argv[1]) + out_dir = Path(sys.argv[2]) + license_file = Path(sys.argv[3]) if len(sys.argv) == 4 else Path(__file__).resolve().parents[2] / "LICENSE" + + cli_version, sums = read_checksums(artifact_dir) + version = pep440(cli_version) + license_text = license_file.read_bytes() + out_dir.mkdir(parents=True, exist_ok=True) + + for (goos, goarch), platform_tag in PLATFORMS.items(): + archive = find_archive(artifact_dir, goos, goarch) + actual = hashlib.sha256(archive.read_bytes()).hexdigest() + if sums.get(archive.name) != actual: + raise SystemExit(f"checksum mismatch for {archive.name}: expected {sums.get(archive.name)}, got {actual}") + binary_name = "databricks.exe" if goos == "windows" else "databricks" + binary = extract_binary(archive, binary_name) + print(build_wheel(out_dir, version, platform_tag, binary_name, binary, license_text)) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/test_build_cli_wheels.py b/.github/scripts/test_build_cli_wheels.py new file mode 100644 index 00000000000..c6bf5f02d93 --- /dev/null +++ b/.github/scripts/test_build_cli_wheels.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import io +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path + +import build_cli_wheels as b + + +class TestBuildCliWheels(unittest.TestCase): + def test_pep440(self): + self.assertEqual(b.pep440("1.23.0"), "1.23.0") + self.assertEqual(b.pep440("1.23.1-dev+abc123"), "1.23.1.dev0+abc123") + with self.assertRaises(SystemExit): + b.pep440("v1.23.0") + + def test_end_to_end(self): + with tempfile.TemporaryDirectory() as tmp: + art, out = Path(tmp, "art"), Path(tmp, "out") + art.mkdir() + sums = [] + for (goos, goarch), _ in b.PLATFORMS.items(): + if goos == "windows": + path = art / f"databricks_cli_1.23.0_{goos}_{goarch}.zip" + with zipfile.ZipFile(path, "w") as z: + z.writestr("databricks.exe", b"MZ") + else: + path = art / f"databricks_cli_1.23.0_{goos}_{goarch}.tar.gz" + with tarfile.open(path, "w:gz") as t: + info = tarfile.TarInfo("databricks") + info.size = 3 + t.addfile(info, io.BytesIO(b"ELF")) + sums.append(f"{b.hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}") + (art / "databricks_cli_1.23.0_SHA256SUMS").write_text("\n".join(sums)) + lic = Path(tmp, "LICENSE") + lic.write_text("license") + + b.sys.argv = ["x", str(art), str(out), str(lic)] + b.main() + + wheels = sorted(p.name for p in out.glob("*.whl")) + self.assertEqual(len(wheels), len(b.PLATFORMS)) + self.assertIn("databricks-1.23.0-py3-none-win_amd64.whl", wheels) + with zipfile.ZipFile(out / "databricks-1.23.0-py3-none-macosx_11_0_arm64.whl") as z: + script = z.getinfo("databricks-1.23.0.data/scripts/databricks") + self.assertEqual(script.external_attr >> 16, 0o100755) + self.assertIn("Tag: py3-none-macosx_11_0_arm64", z.read("databricks-1.23.0.dist-info/WHEEL").decode()) + + def test_checksum_mismatch(self): + with tempfile.TemporaryDirectory() as tmp: + art = Path(tmp) + (art / "databricks_cli_1.23.0_SHA256SUMS").write_text("00 databricks_cli_1.23.0_linux_amd64.tar.gz\n") + (art / "databricks_cli_1.23.0_linux_amd64.tar.gz").write_bytes(b"x") + for name in ("linux_arm64", "darwin_amd64", "darwin_arm64"): + (art / f"databricks_cli_1.23.0_{name}.tar.gz").write_bytes(b"x") + for name in ("windows_amd64", "windows_arm64"): + (art / f"databricks_cli_1.23.0_{name}.zip").write_bytes(b"x") + (art / "LICENSE").write_text("license") + b.sys.argv = ["x", str(art), str(art / "out"), str(art / "LICENSE")] + with self.assertRaisesRegex(SystemExit, "checksum mismatch"): + b.main() + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index d08ca05da2a..320777af163 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -191,3 +191,63 @@ jobs: with: name: wheel path: python/dist/* + + # Wraps the signed binaries from the cli job into PyPI wheels (project + # "databricks"). Kept as a separate artifact from "wheel" (databricks-bundles) + # so each artifact maps to one PyPI project in the databricks-cli release workflow. + cli-wheels: + needs: cli + runs-on: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + permissions: + id-token: write + contents: read + + steps: + # The script and LICENSE come from the workflow's own ref, like the + # local actions in the other jobs, so building an older tag still works. + - name: Checkout workflow ref + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + sparse-checkout: | + .github + LICENSE + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.6.5" + + - name: Download signed CLI binaries + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: cli + path: cli-artifact + + - name: Build wheels + run: python3 .github/scripts/build_cli_wheels.py cli-artifact cli-wheels + + - name: Check wheel metadata + run: uvx twine@6.2.0 check --strict cli-wheels/* + + - name: Smoke test linux-amd64 wheel + run: | + cli_version=$(basename cli-artifact/databricks_cli_*_SHA256SUMS) + cli_version=${cli_version#databricks_cli_} + cli_version=${cli_version%_SHA256SUMS} + uv venv "$RUNNER_TEMP/venv" + uv pip install --python "$RUNNER_TEMP/venv" cli-wheels/*manylinux*x86_64.whl + "$RUNNER_TEMP/venv/bin/databricks" --version | tee /dev/stderr | grep -F "$cli_version" + + - name: Upload CLI wheels + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: cli-wheels + path: cli-wheels/*.whl + if-no-files-found: error diff --git a/Taskfile.yml b/Taskfile.yml index 0c4184c9778..30a3bc9c2a8 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -265,6 +265,12 @@ tasks: cmds: - 'uv run -p ">=3.11" --no-project python tools/validate_codeowners.py' + test-cli-wheels: + desc: Unit-test the script that packages CLI binaries as PyPI wheels + dir: .github/scripts + cmds: + - 'uv run -p ">=3.11" --no-project python -m unittest -q test_build_cli_wheels' + changelog-preview: desc: Print the CHANGELOG.md section the next release would add from .nextchanges/ env: @@ -314,6 +320,7 @@ tasks: - task: check-codeowners - task: check-lockfiles - task: check-agents-setup + - task: test-cli-wheels install-pythons: desc: Install Python 3.9-3.13 via uv From f82ce57c8863f335ea407086783d7632c6509454 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 23 Oct 2026 13:28:18 +0200 Subject: [PATCH 02/10] Add acceptance test running a built CLI wheel through uvx Package the CLI under test with build_cli_wheels.py and run it with `uvx --find-links`, so uv picks the wheel for this platform and the test fails if a wheel is not installable or does not run. Co-authored-by: Isaac --- acceptance/cmd/pypi-wheel/make_archives.py | 25 ++++++++++++++++++++++ acceptance/cmd/pypi-wheel/out.test.toml | 1 + acceptance/cmd/pypi-wheel/output.txt | 10 +++++++++ acceptance/cmd/pypi-wheel/script | 10 +++++++++ acceptance/cmd/pypi-wheel/test.toml | 5 +++++ 5 files changed, 51 insertions(+) create mode 100644 acceptance/cmd/pypi-wheel/make_archives.py create mode 100644 acceptance/cmd/pypi-wheel/out.test.toml create mode 100644 acceptance/cmd/pypi-wheel/output.txt create mode 100644 acceptance/cmd/pypi-wheel/script create mode 100644 acceptance/cmd/pypi-wheel/test.toml diff --git a/acceptance/cmd/pypi-wheel/make_archives.py b/acceptance/cmd/pypi-wheel/make_archives.py new file mode 100644 index 00000000000..418a4dbcb6c --- /dev/null +++ b/acceptance/cmd/pypi-wheel/make_archives.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Fake goreleaser output: every platform archive holds the given CLI binary.""" + +import hashlib +import shutil +import sys +import tarfile +import zipfile +from pathlib import Path + +cli, out = Path(sys.argv[1]), Path(sys.argv[2]) +out.mkdir() +sums = [] +for goos in ("linux", "darwin", "windows"): + for goarch in ("amd64", "arm64"): + if goos == "windows": + path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.zip" + with zipfile.ZipFile(path, "w") as z: + z.write(cli, "databricks.exe") + else: + path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.tar.gz" + with tarfile.open(path, "w:gz") as t: + t.add(cli, "databricks") + sums.append(f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}\n") +(out / "databricks_cli_1.2.3_SHA256SUMS").write_text("".join(sums)) diff --git a/acceptance/cmd/pypi-wheel/out.test.toml b/acceptance/cmd/pypi-wheel/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/cmd/pypi-wheel/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/cmd/pypi-wheel/output.txt b/acceptance/cmd/pypi-wheel/output.txt new file mode 100644 index 00000000000..b9fe8b9ea99 --- /dev/null +++ b/acceptance/cmd/pypi-wheel/output.txt @@ -0,0 +1,10 @@ + +>>> python3 make_archives.py [CLI] archives + +>>> python3 [TESTROOT]/../.github/scripts/build_cli_wheels.py archives wheels [TESTROOT]/../LICENSE + +>>> ls wheels + 6 + +>>> uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version +Databricks CLI v[CLI_VERSION] diff --git a/acceptance/cmd/pypi-wheel/script b/acceptance/cmd/pypi-wheel/script new file mode 100644 index 00000000000..4bd126cc6a9 --- /dev/null +++ b/acceptance/cmd/pypi-wheel/script @@ -0,0 +1,10 @@ +# Package the CLI under test the way release-build does, then run it through uvx +# to check that a wheel built by build_cli_wheels.py is usable by Python tooling. +trace python3 make_archives.py $CLI archives +trace python3 $TESTROOT/../.github/scripts/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE > /dev/null +trace ls wheels | sed -E 's/.*py3-none-.*//' | sort | uniq -c + +# uv picks the wheel matching this platform from the directory. +trace uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version + +rm -fr archives wheels diff --git a/acceptance/cmd/pypi-wheel/test.toml b/acceptance/cmd/pypi-wheel/test.toml new file mode 100644 index 00000000000..23eaa49b0c6 --- /dev/null +++ b/acceptance/cmd/pypi-wheel/test.toml @@ -0,0 +1,5 @@ +Cloud = false + +# uvx may need to create a Python environment. +Timeout = '180s' +TimeoutWindows = '240s' From 5b7776f5eaeb56bcf642413b4b47ca605796eca3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 23 Oct 2026 18:36:06 +0200 Subject: [PATCH 03/10] Move CLI wheel builder to tools/ and fix pypi-wheel test output Python helpers live in tools/. Also drop `uniq -c` from the acceptance test: its padding differs between macOS and Linux/Windows. Co-authored-by: Isaac --- .github/workflows/release-build.yml | 3 ++- Taskfile.yml | 2 +- acceptance/cmd/pypi-wheel/output.txt | 4 ++-- acceptance/cmd/pypi-wheel/script | 4 ++-- {.github/scripts => tools}/build_cli_wheels.py | 2 +- {.github/scripts => tools}/test_build_cli_wheels.py | 0 6 files changed, 8 insertions(+), 7 deletions(-) rename {.github/scripts => tools}/build_cli_wheels.py (99%) rename {.github/scripts => tools}/test_build_cli_wheels.py (100%) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 320777af163..e98963d94c9 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -214,6 +214,7 @@ jobs: ref: ${{ github.sha }} sparse-checkout: | .github + tools/build_cli_wheels.py LICENSE - name: Setup JFrog @@ -231,7 +232,7 @@ jobs: path: cli-artifact - name: Build wheels - run: python3 .github/scripts/build_cli_wheels.py cli-artifact cli-wheels + run: python3 tools/build_cli_wheels.py cli-artifact cli-wheels - name: Check wheel metadata run: uvx twine@6.2.0 check --strict cli-wheels/* diff --git a/Taskfile.yml b/Taskfile.yml index 30a3bc9c2a8..924aa51f9bb 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -267,7 +267,7 @@ tasks: test-cli-wheels: desc: Unit-test the script that packages CLI binaries as PyPI wheels - dir: .github/scripts + dir: tools cmds: - 'uv run -p ">=3.11" --no-project python -m unittest -q test_build_cli_wheels' diff --git a/acceptance/cmd/pypi-wheel/output.txt b/acceptance/cmd/pypi-wheel/output.txt index b9fe8b9ea99..01457a08a26 100644 --- a/acceptance/cmd/pypi-wheel/output.txt +++ b/acceptance/cmd/pypi-wheel/output.txt @@ -1,10 +1,10 @@ >>> python3 make_archives.py [CLI] archives ->>> python3 [TESTROOT]/../.github/scripts/build_cli_wheels.py archives wheels [TESTROOT]/../LICENSE +>>> python3 [TESTROOT]/../tools/build_cli_wheels.py archives wheels [TESTROOT]/../LICENSE >>> ls wheels - 6 +6 >>> uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version Databricks CLI v[CLI_VERSION] diff --git a/acceptance/cmd/pypi-wheel/script b/acceptance/cmd/pypi-wheel/script index 4bd126cc6a9..e87df944008 100644 --- a/acceptance/cmd/pypi-wheel/script +++ b/acceptance/cmd/pypi-wheel/script @@ -1,8 +1,8 @@ # Package the CLI under test the way release-build does, then run it through uvx # to check that a wheel built by build_cli_wheels.py is usable by Python tooling. trace python3 make_archives.py $CLI archives -trace python3 $TESTROOT/../.github/scripts/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE > /dev/null -trace ls wheels | sed -E 's/.*py3-none-.*//' | sort | uniq -c +trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE > /dev/null +trace ls wheels | grep -c '^databricks-1.2.3-py3-none-.*\.whl$' # uv picks the wheel matching this platform from the directory. trace uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version diff --git a/.github/scripts/build_cli_wheels.py b/tools/build_cli_wheels.py similarity index 99% rename from .github/scripts/build_cli_wheels.py rename to tools/build_cli_wheels.py index 6521650d442..b205bd9eec4 100644 --- a/.github/scripts/build_cli_wheels.py +++ b/tools/build_cli_wheels.py @@ -160,7 +160,7 @@ def main() -> None: raise SystemExit(__doc__) artifact_dir = Path(sys.argv[1]) out_dir = Path(sys.argv[2]) - license_file = Path(sys.argv[3]) if len(sys.argv) == 4 else Path(__file__).resolve().parents[2] / "LICENSE" + license_file = Path(sys.argv[3]) if len(sys.argv) == 4 else Path(__file__).resolve().parents[1] / "LICENSE" cli_version, sums = read_checksums(artifact_dir) version = pep440(cli_version) diff --git a/.github/scripts/test_build_cli_wheels.py b/tools/test_build_cli_wheels.py similarity index 100% rename from .github/scripts/test_build_cli_wheels.py rename to tools/test_build_cli_wheels.py From 5126238e34035b13539d0739a6c037a98ab82a31 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 23 Oct 2026 18:54:30 +0200 Subject: [PATCH 04/10] Drop checksum verification from the CLI wheel builder The archives come from the cli artifact of the same workflow run, so the extra check adds nothing. The SHA256SUMS file name still carries the version. Co-authored-by: Isaac --- acceptance/cmd/pypi-wheel/make_archives.py | 7 ++----- tools/build_cli_wheels.py | 18 +++++------------- tools/test_build_cli_wheels.py | 18 +----------------- 3 files changed, 8 insertions(+), 35 deletions(-) diff --git a/acceptance/cmd/pypi-wheel/make_archives.py b/acceptance/cmd/pypi-wheel/make_archives.py index 418a4dbcb6c..e784866ff9c 100644 --- a/acceptance/cmd/pypi-wheel/make_archives.py +++ b/acceptance/cmd/pypi-wheel/make_archives.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 """Fake goreleaser output: every platform archive holds the given CLI binary.""" -import hashlib -import shutil import sys import tarfile import zipfile @@ -10,7 +8,6 @@ cli, out = Path(sys.argv[1]), Path(sys.argv[2]) out.mkdir() -sums = [] for goos in ("linux", "darwin", "windows"): for goarch in ("amd64", "arm64"): if goos == "windows": @@ -21,5 +18,5 @@ path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.tar.gz" with tarfile.open(path, "w:gz") as t: t.add(cli, "databricks") - sums.append(f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}\n") -(out / "databricks_cli_1.2.3_SHA256SUMS").write_text("".join(sums)) +# Only the file name is used: it carries the version. +(out / "databricks_cli_1.2.3_SHA256SUMS").touch() diff --git a/tools/build_cli_wheels.py b/tools/build_cli_wheels.py index b205bd9eec4..afedd541759 100644 --- a/tools/build_cli_wheels.py +++ b/tools/build_cli_wheels.py @@ -2,7 +2,7 @@ """Package the released Databricks CLI binaries as PyPI wheels. Input is the directory of the `cli` release-build artifact: the goreleaser -archives (databricks_cli_*__.{tar.gz,zip}) and the SHA256SUMS file. +archives (databricks_cli_*__.{tar.gz,zip}) and the SHA256SUMS file, whose name carries the version. The binaries are taken from the archives as-is, so the signed Windows binaries are what ends up in the wheels. @@ -52,17 +52,12 @@ ZIP_DATE_TIME = (1980, 1, 1, 0, 0, 0) -def read_checksums(artifact_dir: Path) -> tuple[str, dict[str, str]]: - """Return (cli_version, {file name: sha256}) from the SHA256SUMS file.""" +def read_version(artifact_dir: Path) -> str: + """Return the CLI version from the databricks_cli__SHA256SUMS file name.""" sums_files = list(artifact_dir.glob("databricks_cli_*_SHA256SUMS")) if len(sums_files) != 1: raise SystemExit(f"expected exactly one databricks_cli_*_SHA256SUMS in {artifact_dir}, found {sums_files}") - version = sums_files[0].name[len("databricks_cli_") : -len("_SHA256SUMS")] - sums = {} - for line in sums_files[0].read_text().splitlines(): - digest, _, name = line.partition(" ") - sums[name.strip()] = digest - return version, sums + return sums_files[0].name[len("databricks_cli_") : -len("_SHA256SUMS")] def pep440(cli_version: str) -> str: @@ -162,16 +157,13 @@ def main() -> None: out_dir = Path(sys.argv[2]) license_file = Path(sys.argv[3]) if len(sys.argv) == 4 else Path(__file__).resolve().parents[1] / "LICENSE" - cli_version, sums = read_checksums(artifact_dir) + cli_version = read_version(artifact_dir) version = pep440(cli_version) license_text = license_file.read_bytes() out_dir.mkdir(parents=True, exist_ok=True) for (goos, goarch), platform_tag in PLATFORMS.items(): archive = find_archive(artifact_dir, goos, goarch) - actual = hashlib.sha256(archive.read_bytes()).hexdigest() - if sums.get(archive.name) != actual: - raise SystemExit(f"checksum mismatch for {archive.name}: expected {sums.get(archive.name)}, got {actual}") binary_name = "databricks.exe" if goos == "windows" else "databricks" binary = extract_binary(archive, binary_name) print(build_wheel(out_dir, version, platform_tag, binary_name, binary, license_text)) diff --git a/tools/test_build_cli_wheels.py b/tools/test_build_cli_wheels.py index c6bf5f02d93..1086c0ea922 100644 --- a/tools/test_build_cli_wheels.py +++ b/tools/test_build_cli_wheels.py @@ -20,7 +20,6 @@ def test_end_to_end(self): with tempfile.TemporaryDirectory() as tmp: art, out = Path(tmp, "art"), Path(tmp, "out") art.mkdir() - sums = [] for (goos, goarch), _ in b.PLATFORMS.items(): if goos == "windows": path = art / f"databricks_cli_1.23.0_{goos}_{goarch}.zip" @@ -32,8 +31,7 @@ def test_end_to_end(self): info = tarfile.TarInfo("databricks") info.size = 3 t.addfile(info, io.BytesIO(b"ELF")) - sums.append(f"{b.hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}") - (art / "databricks_cli_1.23.0_SHA256SUMS").write_text("\n".join(sums)) + (art / "databricks_cli_1.23.0_SHA256SUMS").touch() lic = Path(tmp, "LICENSE") lic.write_text("license") @@ -48,20 +46,6 @@ def test_end_to_end(self): self.assertEqual(script.external_attr >> 16, 0o100755) self.assertIn("Tag: py3-none-macosx_11_0_arm64", z.read("databricks-1.23.0.dist-info/WHEEL").decode()) - def test_checksum_mismatch(self): - with tempfile.TemporaryDirectory() as tmp: - art = Path(tmp) - (art / "databricks_cli_1.23.0_SHA256SUMS").write_text("00 databricks_cli_1.23.0_linux_amd64.tar.gz\n") - (art / "databricks_cli_1.23.0_linux_amd64.tar.gz").write_bytes(b"x") - for name in ("linux_arm64", "darwin_amd64", "darwin_arm64"): - (art / f"databricks_cli_1.23.0_{name}.tar.gz").write_bytes(b"x") - for name in ("windows_amd64", "windows_arm64"): - (art / f"databricks_cli_1.23.0_{name}.zip").write_bytes(b"x") - (art / "LICENSE").write_text("license") - b.sys.argv = ["x", str(art), str(art / "out"), str(art / "LICENSE")] - with self.assertRaisesRegex(SystemExit, "checksum mismatch"): - b.main() - if __name__ == "__main__": unittest.main() From 050f0bc1f1a4220531872fe948d56fe68f52ba4d Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Mon, 16 Nov 2026 19:49:55 +0000 Subject: [PATCH 05/10] CLI wheels: tag macOS 13, emit Metadata-Version 2.4, align action pins The Go 1.27 darwin binaries have LC_BUILD_VERSION minos 13.0, so a macosx_11_0 tag would let pip install a binary that cannot start on macOS 11 and 12. License-File is only valid from Metadata-Version 2.4, which packaging.metadata (and so PyPI) enforces. Co-authored-by: Isaac --- .github/workflows/release-build.yml | 4 ++-- tools/build_cli_wheels.py | 13 ++++++++----- tools/test_build_cli_wheels.py | 7 +++++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index e98963d94c9..5d18a9d458b 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -221,7 +221,7 @@ jobs: uses: ./.github/actions/setup-jfrog - name: Install uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: version: "0.6.5" @@ -247,7 +247,7 @@ jobs: "$RUNNER_TEMP/venv/bin/databricks" --version | tee /dev/stderr | grep -F "$cli_version" - name: Upload CLI wheels - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: cli-wheels path: cli-wheels/*.whl diff --git a/tools/build_cli_wheels.py b/tools/build_cli_wheels.py index afedd541759..f619c920aaa 100644 --- a/tools/build_cli_wheels.py +++ b/tools/build_cli_wheels.py @@ -42,8 +42,10 @@ PLATFORMS = { ("linux", "amd64"): "manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64", ("linux", "arm64"): "manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64", - ("darwin", "amd64"): "macosx_11_0_x86_64", - ("darwin", "arm64"): "macosx_11_0_arm64", + # Go 1.27 binaries require macOS 13 (LC_BUILD_VERSION minos); a lower tag + # would install a binary that cannot start on older macOS. + ("darwin", "amd64"): "macosx_13_0_x86_64", + ("darwin", "arm64"): "macosx_13_0_arm64", ("windows", "amd64"): "win_amd64", ("windows", "arm64"): "win_arm64", } @@ -111,12 +113,13 @@ def build_wheel( data_dir = f"{DIST_NAME}-{version}.data" metadata = ( - "Metadata-Version: 2.1\n" + # 2.4 is the first version that allows License-File; PyPI validates this. + "Metadata-Version: 2.4\n" f"Name: {DIST_NAME}\n" f"Version: {version}\n" f"Summary: {SUMMARY}\n" - "Home-page: https://github.com/databricks/cli\n" - "License: Databricks License\n" + "Project-URL: Source, https://github.com/databricks/cli\n" + "Project-URL: Documentation, https://docs.databricks.com/dev-tools/cli/\n" "License-File: LICENSE\n" "Description-Content-Type: text/markdown\n" f"\n{DESCRIPTION}" diff --git a/tools/test_build_cli_wheels.py b/tools/test_build_cli_wheels.py index 1086c0ea922..b7d75d63748 100644 --- a/tools/test_build_cli_wheels.py +++ b/tools/test_build_cli_wheels.py @@ -41,10 +41,13 @@ def test_end_to_end(self): wheels = sorted(p.name for p in out.glob("*.whl")) self.assertEqual(len(wheels), len(b.PLATFORMS)) self.assertIn("databricks-1.23.0-py3-none-win_amd64.whl", wheels) - with zipfile.ZipFile(out / "databricks-1.23.0-py3-none-macosx_11_0_arm64.whl") as z: + with zipfile.ZipFile(out / "databricks-1.23.0-py3-none-macosx_13_0_arm64.whl") as z: script = z.getinfo("databricks-1.23.0.data/scripts/databricks") self.assertEqual(script.external_attr >> 16, 0o100755) - self.assertIn("Tag: py3-none-macosx_11_0_arm64", z.read("databricks-1.23.0.dist-info/WHEEL").decode()) + self.assertIn("Tag: py3-none-macosx_13_0_arm64", z.read("databricks-1.23.0.dist-info/WHEEL").decode()) + metadata = z.read("databricks-1.23.0.dist-info/METADATA").decode() + self.assertTrue(metadata.startswith("Metadata-Version: 2.4\n")) + self.assertIn("License-File: LICENSE\n", metadata) if __name__ == "__main__": From 9b9c675fdc450c811cd0971761d295eb9f5fc40f Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 11:34:32 +0000 Subject: [PATCH 06/10] pypi-wheel test: fail if uv installs another platform's wheel; uv 0.8.9 in cli-wheels Only the archive for the test's own platform holds the CLI binary; the others hold a placeholder, so running the installed databricks fails if uv picks the wrong wheel. The cli-wheels job now uses the uv version the test jobs use. Co-authored-by: Isaac --- .github/workflows/release-build.yml | 2 +- acceptance/cmd/pypi-wheel/make_archives.py | 23 +++++++++++++++++++--- acceptance/cmd/pypi-wheel/script | 3 ++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 5d18a9d458b..562bed83f9e 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -223,7 +223,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: - version: "0.6.5" + version: "0.8.9" - name: Download signed CLI binaries uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 diff --git a/acceptance/cmd/pypi-wheel/make_archives.py b/acceptance/cmd/pypi-wheel/make_archives.py index e784866ff9c..9b751188b0f 100644 --- a/acceptance/cmd/pypi-wheel/make_archives.py +++ b/acceptance/cmd/pypi-wheel/make_archives.py @@ -1,22 +1,39 @@ #!/usr/bin/env python3 -"""Fake goreleaser output: every platform archive holds the given CLI binary.""" +"""Fake goreleaser output for the CLI wheel builder. +Only the archive for the platform running the test holds the CLI binary. The +others hold a placeholder that cannot be executed, so running the installed +`databricks` fails if uv picks another platform's wheel. +""" + +import io +import platform import sys import tarfile import zipfile from pathlib import Path +GOOS = {"Linux": "linux", "Darwin": "darwin", "Windows": "windows"} +GOARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"} + cli, out = Path(sys.argv[1]), Path(sys.argv[2]) +host = (GOOS[platform.system()], GOARCH[platform.machine().lower()]) +placeholder = b"placeholder for a binary of another platform\n" + out.mkdir() for goos in ("linux", "darwin", "windows"): for goarch in ("amd64", "arm64"): + data = cli.read_bytes() if (goos, goarch) == host else placeholder if goos == "windows": path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.zip" with zipfile.ZipFile(path, "w") as z: - z.write(cli, "databricks.exe") + z.writestr("databricks.exe", data) else: path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.tar.gz" with tarfile.open(path, "w:gz") as t: - t.add(cli, "databricks") + info = tarfile.TarInfo("databricks") + info.size = len(data) + info.mode = 0o755 + t.addfile(info, io.BytesIO(data)) # Only the file name is used: it carries the version. (out / "databricks_cli_1.2.3_SHA256SUMS").touch() diff --git a/acceptance/cmd/pypi-wheel/script b/acceptance/cmd/pypi-wheel/script index e87df944008..daabd6339eb 100644 --- a/acceptance/cmd/pypi-wheel/script +++ b/acceptance/cmd/pypi-wheel/script @@ -4,7 +4,8 @@ trace python3 make_archives.py $CLI archives trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE > /dev/null trace ls wheels | grep -c '^databricks-1.2.3-py3-none-.*\.whl$' -# uv picks the wheel matching this platform from the directory. +# uv must pick the wheel for this platform: the other wheels hold a placeholder +# instead of the binary (see make_archives.py), so a wrong pick fails here. trace uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version rm -fr archives wheels From 06abe51fbe2ca8abab445e4fae15e7eb7a401fc9 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 11:48:38 +0000 Subject: [PATCH 07/10] pypi-wheel test: record archives and wheel contents; skip gzip in fake archives The test now shows the archive and wheel names, the files and modes in each wheel, the WHEEL tags and METADATA, so changes to the wheel layout show up in review. Writing the fake archives uncompressed takes the test from about 12s to 4s; uvx itself takes well under a second. Co-authored-by: Isaac --- acceptance/cmd/pypi-wheel/make_archives.py | 3 +- acceptance/cmd/pypi-wheel/output.txt | 100 ++++++++++++++++++++- acceptance/cmd/pypi-wheel/script | 5 +- acceptance/cmd/pypi-wheel/show_wheels.py | 23 +++++ 4 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 acceptance/cmd/pypi-wheel/show_wheels.py diff --git a/acceptance/cmd/pypi-wheel/make_archives.py b/acceptance/cmd/pypi-wheel/make_archives.py index 9b751188b0f..3d1867700d9 100644 --- a/acceptance/cmd/pypi-wheel/make_archives.py +++ b/acceptance/cmd/pypi-wheel/make_archives.py @@ -30,7 +30,8 @@ z.writestr("databricks.exe", data) else: path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.tar.gz" - with tarfile.open(path, "w:gz") as t: + # Compression only costs test time: the builder reads the archive either way. + with tarfile.open(path, "w:gz", compresslevel=0) as t: info = tarfile.TarInfo("databricks") info.size = len(data) info.mode = 0o755 diff --git a/acceptance/cmd/pypi-wheel/output.txt b/acceptance/cmd/pypi-wheel/output.txt index 01457a08a26..3aa93f52067 100644 --- a/acceptance/cmd/pypi-wheel/output.txt +++ b/acceptance/cmd/pypi-wheel/output.txt @@ -1,10 +1,108 @@ >>> python3 make_archives.py [CLI] archives +>>> ls archives +databricks_cli_1.2.3_SHA256SUMS +databricks_cli_1.2.3_darwin_amd64.tar.gz +databricks_cli_1.2.3_darwin_arm64.tar.gz +databricks_cli_1.2.3_linux_amd64.tar.gz +databricks_cli_1.2.3_linux_arm64.tar.gz +databricks_cli_1.2.3_windows_amd64.zip +databricks_cli_1.2.3_windows_arm64.zip + >>> python3 [TESTROOT]/../tools/build_cli_wheels.py archives wheels [TESTROOT]/../LICENSE >>> ls wheels -6 +databricks-1.2.3-py3-none-macosx_13_0_arm64.whl +databricks-1.2.3-py3-none-macosx_13_0_x86_64.whl +databricks-1.2.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl +databricks-1.2.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64.whl +databricks-1.2.3-py3-none-win_amd64.whl +databricks-1.2.3-py3-none-win_arm64.whl + +>>> python3 show_wheels.py wheels +== databricks-1.2.3-py3-none-macosx_13_0_arm64.whl +100755 databricks-1.2.3.data/scripts/databricks +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-macosx_13_0_arm64 +== databricks-1.2.3-py3-none-macosx_13_0_x86_64.whl +100755 databricks-1.2.3.data/scripts/databricks +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-macosx_13_0_x86_64 +== databricks-1.2.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl +100755 databricks-1.2.3.data/scripts/databricks +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-manylinux_2_17_aarch64 +Tag: py3-none-manylinux2014_aarch64 +Tag: py3-none-musllinux_1_1_aarch64 +== databricks-1.2.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64.whl +100755 databricks-1.2.3.data/scripts/databricks +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-manylinux_2_17_x86_64 +Tag: py3-none-manylinux2014_x86_64 +Tag: py3-none-musllinux_1_1_x86_64 +== databricks-1.2.3-py3-none-win_amd64.whl +100755 databricks-1.2.3.data/scripts/databricks.exe +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-win_amd64 +== databricks-1.2.3-py3-none-win_arm64.whl +100755 databricks-1.2.3.data/scripts/databricks.exe +100644 databricks-1.2.3.dist-info/METADATA +100644 databricks-1.2.3.dist-info/WHEEL +100644 databricks-1.2.3.dist-info/licenses/LICENSE +100644 databricks-1.2.3.dist-info/RECORD +Wheel-Version: 1.0 +Generator: databricks-cli build_cli_wheels.py +Root-Is-Purelib: false +Tag: py3-none-win_arm64 +== METADATA (same in every wheel) +Metadata-Version: 2.4 +Name: databricks +Version: 1.2.3 +Summary: Databricks CLI +Project-URL: Source, https://github.com/databricks/cli +Project-URL: Documentation, https://docs.databricks.com/dev-tools/cli/ +License-File: LICENSE +Description-Content-Type: text/markdown + +# Databricks CLI + +Command-line interface for Databricks. This package installs the prebuilt `databricks` binary. + +Run it with `uvx databricks` or `pipx run databricks`, or install it with `uv tool install databricks`. + +Documentation: https://docs.databricks.com/dev-tools/cli/ +Source: https://github.com/databricks/cli >>> uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version Databricks CLI v[CLI_VERSION] diff --git a/acceptance/cmd/pypi-wheel/script b/acceptance/cmd/pypi-wheel/script index daabd6339eb..9df8b5f8d93 100644 --- a/acceptance/cmd/pypi-wheel/script +++ b/acceptance/cmd/pypi-wheel/script @@ -1,8 +1,11 @@ # Package the CLI under test the way release-build does, then run it through uvx # to check that a wheel built by build_cli_wheels.py is usable by Python tooling. trace python3 make_archives.py $CLI archives +trace ls archives +# The script prints the wheel paths; they are listed below with ls, which also works on Windows. trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE > /dev/null -trace ls wheels | grep -c '^databricks-1.2.3-py3-none-.*\.whl$' +trace ls wheels +trace python3 show_wheels.py wheels # uv must pick the wheel for this platform: the other wheels hold a placeholder # instead of the binary (see make_archives.py), so a wrong pick fails here. diff --git a/acceptance/cmd/pypi-wheel/show_wheels.py b/acceptance/cmd/pypi-wheel/show_wheels.py new file mode 100644 index 00000000000..53ba6090b4a --- /dev/null +++ b/acceptance/cmd/pypi-wheel/show_wheels.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Print what each wheel in a directory installs, in a platform-independent form. + +RECORD is not printed: its hashes cover the CLI binary, which differs per build. +""" + +import sys +import zipfile +from pathlib import Path + +wheels = sorted(Path(sys.argv[1]).glob("*.whl")) +for whl in wheels: + print(f"== {whl.name}") + with zipfile.ZipFile(whl) as z: + for info in z.infolist(): + print(f"{info.external_attr >> 16:o} {info.filename}") + wheel_file = next(n for n in z.namelist() if n.endswith(".dist-info/WHEEL")) + print(z.read(wheel_file).decode(), end="") + +with zipfile.ZipFile(wheels[0]) as z: + metadata_file = next(n for n in z.namelist() if n.endswith(".dist-info/METADATA")) + print("== METADATA (same in every wheel)") + print(z.read(metadata_file).decode(), end="") From ae1ecec0f029d8b2dc1a7ff1e41afa85f289b3ef Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 11:56:16 +0000 Subject: [PATCH 08/10] build_cli_wheels: print wheel paths with forward slashes Lets the pypi-wheel acceptance test record the builder's output, which is then the same on Windows. Co-authored-by: Isaac --- acceptance/cmd/pypi-wheel/output.txt | 14 ++++++-------- acceptance/cmd/pypi-wheel/script | 4 +--- tools/build_cli_wheels.py | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/acceptance/cmd/pypi-wheel/output.txt b/acceptance/cmd/pypi-wheel/output.txt index 3aa93f52067..e63bc6cabda 100644 --- a/acceptance/cmd/pypi-wheel/output.txt +++ b/acceptance/cmd/pypi-wheel/output.txt @@ -11,14 +11,12 @@ databricks_cli_1.2.3_windows_amd64.zip databricks_cli_1.2.3_windows_arm64.zip >>> python3 [TESTROOT]/../tools/build_cli_wheels.py archives wheels [TESTROOT]/../LICENSE - ->>> ls wheels -databricks-1.2.3-py3-none-macosx_13_0_arm64.whl -databricks-1.2.3-py3-none-macosx_13_0_x86_64.whl -databricks-1.2.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl -databricks-1.2.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64.whl -databricks-1.2.3-py3-none-win_amd64.whl -databricks-1.2.3-py3-none-win_arm64.whl +wheels/databricks-1.2.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64.whl +wheels/databricks-1.2.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl +wheels/databricks-1.2.3-py3-none-macosx_13_0_x86_64.whl +wheels/databricks-1.2.3-py3-none-macosx_13_0_arm64.whl +wheels/databricks-1.2.3-py3-none-win_amd64.whl +wheels/databricks-1.2.3-py3-none-win_arm64.whl >>> python3 show_wheels.py wheels == databricks-1.2.3-py3-none-macosx_13_0_arm64.whl diff --git a/acceptance/cmd/pypi-wheel/script b/acceptance/cmd/pypi-wheel/script index 9df8b5f8d93..c5dfd3e650a 100644 --- a/acceptance/cmd/pypi-wheel/script +++ b/acceptance/cmd/pypi-wheel/script @@ -2,9 +2,7 @@ # to check that a wheel built by build_cli_wheels.py is usable by Python tooling. trace python3 make_archives.py $CLI archives trace ls archives -# The script prints the wheel paths; they are listed below with ls, which also works on Windows. -trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE > /dev/null -trace ls wheels +trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE trace python3 show_wheels.py wheels # uv must pick the wheel for this platform: the other wheels hold a placeholder diff --git a/tools/build_cli_wheels.py b/tools/build_cli_wheels.py index f619c920aaa..b2cf5a5d026 100644 --- a/tools/build_cli_wheels.py +++ b/tools/build_cli_wheels.py @@ -169,7 +169,7 @@ def main() -> None: archive = find_archive(artifact_dir, goos, goarch) binary_name = "databricks.exe" if goos == "windows" else "databricks" binary = extract_binary(archive, binary_name) - print(build_wheel(out_dir, version, platform_tag, binary_name, binary, license_text)) + print(build_wheel(out_dir, version, platform_tag, binary_name, binary, license_text).as_posix()) if __name__ == "__main__": From 58180a2f61974601382a99797d9410285c0f3f5d Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 12:00:37 +0000 Subject: [PATCH 09/10] Validate CLI wheels with packaging in tests and release-build tools/validate_cli_wheels.py checks what a build backend would otherwise guarantee: the filename, METADATA via packaging (stricter than twine check; it catches License-File with Metadata-Version 2.1), WHEEL tags against the filename, RECORD hashes and sizes, that each wheel holds only the CLI binary, and one wheel per platform with one version. It runs in the pypi-wheel acceptance test (offline, with the vendored packaging), in release-build after twine check, and has unit tests for each failure. Builder messages now use forward slashes too, so outputs match on Windows. Co-authored-by: Isaac --- .github/workflows/release-build.yml | 4 + Taskfile.yml | 4 +- acceptance/cmd/pypi-wheel/output.txt | 8 ++ acceptance/cmd/pypi-wheel/script | 2 + tools/build_cli_wheels.py | 15 ++- tools/test_validate_cli_wheels.py | 97 ++++++++++++++++++ tools/validate_cli_wheels.py | 147 +++++++++++++++++++++++++++ 7 files changed, 272 insertions(+), 5 deletions(-) create mode 100644 tools/test_validate_cli_wheels.py create mode 100644 tools/validate_cli_wheels.py diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 562bed83f9e..2978bc349b2 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -215,6 +215,7 @@ jobs: sparse-checkout: | .github tools/build_cli_wheels.py + tools/validate_cli_wheels.py LICENSE - name: Setup JFrog @@ -237,6 +238,9 @@ jobs: - name: Check wheel metadata run: uvx twine@6.2.0 check --strict cli-wheels/* + - name: Validate wheels + run: uv run -q --no-project --with packaging==25.0 python tools/validate_cli_wheels.py cli-wheels + - name: Smoke test linux-amd64 wheel run: | cli_version=$(basename cli-artifact/databricks_cli_*_SHA256SUMS) diff --git a/Taskfile.yml b/Taskfile.yml index 924aa51f9bb..4dc3fa13e0d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -266,10 +266,10 @@ tasks: - 'uv run -p ">=3.11" --no-project python tools/validate_codeowners.py' test-cli-wheels: - desc: Unit-test the script that packages CLI binaries as PyPI wheels + desc: Unit-test the scripts that package CLI binaries as PyPI wheels and validate them dir: tools cmds: - - 'uv run -p ">=3.11" --no-project python -m unittest -q test_build_cli_wheels' + - 'uv run -p ">=3.11" --no-project --with packaging==25.0 python -m unittest -q test_build_cli_wheels test_validate_cli_wheels' changelog-preview: desc: Print the CHANGELOG.md section the next release would add from .nextchanges/ diff --git a/acceptance/cmd/pypi-wheel/output.txt b/acceptance/cmd/pypi-wheel/output.txt index e63bc6cabda..6350374bf61 100644 --- a/acceptance/cmd/pypi-wheel/output.txt +++ b/acceptance/cmd/pypi-wheel/output.txt @@ -18,6 +18,14 @@ wheels/databricks-1.2.3-py3-none-macosx_13_0_arm64.whl wheels/databricks-1.2.3-py3-none-win_amd64.whl wheels/databricks-1.2.3-py3-none-win_arm64.whl +>>> uv run -q --no-project --with packaging==25.0 python [TESTROOT]/../tools/validate_cli_wheels.py wheels +valid: wheels/databricks-1.2.3-py3-none-macosx_13_0_arm64.whl +valid: wheels/databricks-1.2.3-py3-none-macosx_13_0_x86_64.whl +valid: wheels/databricks-1.2.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl +valid: wheels/databricks-1.2.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64.whl +valid: wheels/databricks-1.2.3-py3-none-win_amd64.whl +valid: wheels/databricks-1.2.3-py3-none-win_arm64.whl + >>> python3 show_wheels.py wheels == databricks-1.2.3-py3-none-macosx_13_0_arm64.whl 100755 databricks-1.2.3.data/scripts/databricks diff --git a/acceptance/cmd/pypi-wheel/script b/acceptance/cmd/pypi-wheel/script index c5dfd3e650a..ddc32543993 100644 --- a/acceptance/cmd/pypi-wheel/script +++ b/acceptance/cmd/pypi-wheel/script @@ -3,6 +3,8 @@ trace python3 make_archives.py $CLI archives trace ls archives trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE +# packaging comes from the vendored packages; the test runs offline. +trace uv run -q --no-project --with packaging==25.0 python $TESTROOT/../tools/validate_cli_wheels.py wheels trace python3 show_wheels.py wheels # uv must pick the wheel for this platform: the other wheels hold a placeholder diff --git a/tools/build_cli_wheels.py b/tools/build_cli_wheels.py index b2cf5a5d026..8bff20792ca 100644 --- a/tools/build_cli_wheels.py +++ b/tools/build_cli_wheels.py @@ -54,11 +54,18 @@ ZIP_DATE_TIME = (1980, 1, 1, 0, 0, 0) +def posix_list(paths: list[Path]) -> list[str]: + """Paths for messages, with forward slashes so output is the same on Windows.""" + return sorted(p.as_posix() for p in paths) + + def read_version(artifact_dir: Path) -> str: """Return the CLI version from the databricks_cli__SHA256SUMS file name.""" sums_files = list(artifact_dir.glob("databricks_cli_*_SHA256SUMS")) if len(sums_files) != 1: - raise SystemExit(f"expected exactly one databricks_cli_*_SHA256SUMS in {artifact_dir}, found {sums_files}") + raise SystemExit( + f"expected exactly one databricks_cli_*_SHA256SUMS in {artifact_dir.as_posix()}, found {posix_list(sums_files)}" + ) return sums_files[0].name[len("databricks_cli_") : -len("_SHA256SUMS")] @@ -81,7 +88,9 @@ def find_archive(artifact_dir: Path, goos: str, goarch: str) -> Path: ext = "zip" if goos == "windows" else "tar.gz" matches = list(artifact_dir.glob(f"databricks_cli_*{goos}_{goarch}.{ext}")) if len(matches) != 1: - raise SystemExit(f"expected exactly one {goos}/{goarch} .{ext} archive in {artifact_dir}, found {matches}") + raise SystemExit( + f"expected exactly one {goos}/{goarch} .{ext} archive in {artifact_dir.as_posix()}, found {posix_list(matches)}" + ) return matches[0] @@ -92,7 +101,7 @@ def extract_binary(archive: Path, binary_name: str) -> bytes: with tarfile.open(archive) as t: f = t.extractfile(binary_name) if f is None: - raise SystemExit(f"{binary_name} is not a regular file in {archive}") + raise SystemExit(f"{binary_name} is not a regular file in {archive.as_posix()}") return f.read() diff --git a/tools/test_validate_cli_wheels.py b/tools/test_validate_cli_wheels.py new file mode 100644 index 00000000000..baf183468c2 --- /dev/null +++ b/tools/test_validate_cli_wheels.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +import tempfile +import unittest +import zipfile +from pathlib import Path + +import build_cli_wheels as b +import validate_cli_wheels as v + +VERSION = "1.23.0" +LINUX_WHEEL = f"databricks-{VERSION}-py3-none-{b.PLATFORMS[('linux', 'amd64')]}.whl" +DIST_INFO = f"databricks-{VERSION}.dist-info" +SCRIPT = f"databricks-{VERSION}.data/scripts/databricks" + + +def build_wheels(out: Path) -> None: + out.mkdir() + for (goos, _), tag in b.PLATFORMS.items(): + binary_name = "databricks.exe" if goos == "windows" else "databricks" + b.build_wheel(out, VERSION, tag, binary_name, b"binary", b"license") + + +def rewrite(whl: Path, changes: dict, fix_record: bool = True, new_name: str = "") -> None: + """Rewrite a wheel with files replaced (bytes) or removed (None), optionally keeping RECORD consistent.""" + with zipfile.ZipFile(whl) as z: + files = {i.filename: (i, z.read(i)) for i in z.infolist()} + for name, data in changes.items(): + if data is None: + del files[name] + else: + info = files[name][0] if name in files else zipfile.ZipInfo(name, b.ZIP_DATE_TIME) + files[name] = (info, data) + if fix_record: + record_name = f"{DIST_INFO}/RECORD" + record = "".join(f"{n},{b.record_hash(d)}\n" for n, (_, d) in files.items() if n != record_name) + files[record_name] = (files[record_name][0], (record + f"{record_name},,\n").encode()) + whl.unlink() + with zipfile.ZipFile(whl.with_name(new_name or whl.name), "w", zipfile.ZIP_DEFLATED) as z: + for info, data in files.values(): + z.writestr(info, data) + + +class TestValidateCliWheels(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.out = Path(tmp.name, "wheels") + build_wheels(self.out) + self.linux = self.out / LINUX_WHEEL + + def assert_invalid(self, message: str): + with self.assertRaises(SystemExit) as cm: + v.validate_dir(self.out) + self.assertIn(message, str(cm.exception)) + + def test_valid(self): + self.assertEqual(len(v.validate_dir(self.out)), 6) + + def test_metadata_version_too_old_for_license_file(self): + with zipfile.ZipFile(self.linux) as z: + metadata = z.read(f"{DIST_INFO}/METADATA").replace(b"Metadata-Version: 2.4", b"Metadata-Version: 2.1") + rewrite(self.linux, {f"{DIST_INFO}/METADATA": metadata}) + self.assert_invalid("license-file introduced in metadata version 2.4") + + def test_metadata_version_mismatch(self): + with zipfile.ZipFile(self.linux) as z: + metadata = z.read(f"{DIST_INFO}/METADATA").replace(b"Version: 1.23.0", b"Version: 1.23.1") + rewrite(self.linux, {f"{DIST_INFO}/METADATA": metadata}) + self.assert_invalid("do not match the filename") + + def test_record_hash_mismatch(self): + rewrite(self.linux, {SCRIPT: b"tampered"}, fix_record=False) + self.assert_invalid(f"RECORD: wrong hash or size for {SCRIPT}") + + def test_extra_file(self): + rewrite(self.linux, {"evil.pth": b"import os"}) + self.assert_invalid("unexpected files: ['evil.pth']") + + def test_missing_license(self): + rewrite(self.linux, {f"{DIST_INFO}/licenses/LICENSE": None}) + self.assert_invalid("unexpected files") + + def test_wheel_tags_differ_from_filename(self): + rewrite(self.linux, {}, new_name=f"databricks-{VERSION}-py3-none-manylinux_2_28_x86_64.whl") + self.assert_invalid("do not match the filename") + + def test_missing_platform(self): + next(self.out.glob("*win_arm64.whl")).unlink() + self.assert_invalid("missing wheels for [('windows', 'arm64')]") + + def test_error_paths_use_forward_slashes(self): + rewrite(self.linux, {"evil.pth": b"import os"}) + self.assert_invalid(f"{self.linux.as_posix()}: ") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/validate_cli_wheels.py b/tools/validate_cli_wheels.py new file mode 100644 index 00000000000..dbc4d57c88e --- /dev/null +++ b/tools/validate_cli_wheels.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Validate the Databricks CLI wheels built by build_cli_wheels.py. + +Checks what a build backend would otherwise guarantee and what pip, uv and +PyPI rely on: the filename, METADATA (validated by `packaging`, which is +stricter than `twine check`), WHEEL tags, RECORD hashes and sizes, and that +each wheel installs only the CLI binary. Also requires exactly one wheel per +supported platform, all with the same version. + +Requires `packaging`. Usage: validate_cli_wheels.py +""" + +import base64 +import csv +import email.parser +import hashlib +import io +import sys +import zipfile +from pathlib import Path + +from packaging.metadata import Metadata +from packaging.utils import parse_wheel_filename + +DIST_NAME = "databricks" +EXPECTED_PLATFORMS = {(goos, goarch) for goos in ("linux", "darwin", "windows") for goarch in ("amd64", "arm64")} +OS_PREFIXES = (("manylinux", "linux"), ("musllinux", "linux"), ("macosx", "darwin"), ("win", "windows")) +ARCH_SUFFIXES = (("x86_64", "amd64"), ("amd64", "amd64"), ("aarch64", "arm64"), ("arm64", "arm64")) + + +class InvalidWheel(Exception): + pass + + +def platform_of(tag: str) -> tuple[str, str]: + goos = next((o for prefix, o in OS_PREFIXES if tag.startswith(prefix)), None) + goarch = next((a for suffix, a in ARCH_SUFFIXES if tag.endswith(suffix)), None) + if goos is None or goarch is None: + raise InvalidWheel(f"unsupported platform tag {tag}") + return goos, goarch + + +def record_hash(data: bytes) -> str: + return "sha256=" + base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + + +def check_record(z: zipfile.ZipFile, record_name: str) -> None: + rows = list(csv.reader(io.StringIO(z.read(record_name).decode()))) + listed = {} + for row in rows: + if len(row) != 3: + raise InvalidWheel(f"RECORD: malformed row {row}") + if row[0] in listed: + raise InvalidWheel(f"RECORD: {row[0]} listed twice") + listed[row[0]] = row + names = z.namelist() + if len(names) != len(set(names)): + raise InvalidWheel("duplicate file in the zip") + if set(listed) != set(names): + raise InvalidWheel(f"RECORD does not match the zip: {sorted(set(listed) ^ set(names))}") + for name, (_, digest, size) in listed.items(): + if name == record_name: + if digest or size: + raise InvalidWheel("RECORD must list itself without a hash or size") + continue + data = z.read(name) + if digest != record_hash(data) or size != str(len(data)): + raise InvalidWheel(f"RECORD: wrong hash or size for {name}") + + +def validate_wheel(path: Path) -> tuple[str, tuple[str, str]]: + """Validate one wheel; return its version and (goos, goarch).""" + name, version, build, tags = parse_wheel_filename(path.name) + if name != DIST_NAME or build: + raise InvalidWheel(f"unexpected name or build tag in {path.name}") + if {(t.interpreter, t.abi) for t in tags} != {("py3", "none")}: + raise InvalidWheel(f"expected py3-none tags, got {sorted(map(str, tags))}") + platforms = {platform_of(t.platform) for t in tags} + if len(platforms) != 1: + raise InvalidWheel(f"tags cover more than one platform: {sorted(platforms)}") + platform = platforms.pop() + + dist_info = f"{DIST_NAME}-{version}.dist-info" + script = f"{DIST_NAME}-{version}.data/scripts/" + ("databricks.exe" if platform[0] == "windows" else "databricks") + expected_files = { + script, + f"{dist_info}/METADATA", + f"{dist_info}/WHEEL", + f"{dist_info}/RECORD", + f"{dist_info}/licenses/LICENSE", + } + + with zipfile.ZipFile(path) as z: + if set(z.namelist()) != expected_files: + raise InvalidWheel(f"unexpected files: {sorted(set(z.namelist()) ^ expected_files)}") + if z.getinfo(script).external_attr >> 16 != 0o100755: + raise InvalidWheel(f"{script} is not a regular file with mode 755") + check_record(z, f"{dist_info}/RECORD") + + try: + metadata = Metadata.from_email(z.read(f"{dist_info}/METADATA"), validate=True) + except Exception as e: + details = "; ".join(str(x) for x in getattr(e, "exceptions", [e])) + raise InvalidWheel(f"METADATA: {details}") from None + if metadata.name != DIST_NAME or metadata.version != version: + raise InvalidWheel(f"METADATA name/version {metadata.name} {metadata.version} do not match the filename") + if metadata.license_files != ["LICENSE"]: + raise InvalidWheel(f"METADATA: expected License-File: LICENSE, got {metadata.license_files}") + + wheel = email.parser.BytesParser().parsebytes(z.read(f"{dist_info}/WHEEL")) + if wheel["Wheel-Version"] != "1.0" or wheel["Root-Is-Purelib"] != "false": + raise InvalidWheel("WHEEL: expected Wheel-Version 1.0 and Root-Is-Purelib false") + if set(wheel.get_all("Tag", [])) != {str(t) for t in tags}: + raise InvalidWheel(f"WHEEL tags {wheel.get_all('Tag')} do not match the filename") + + return str(version), platform + + +def validate_dir(wheels_dir: Path) -> list[Path]: + wheels = sorted(wheels_dir.glob("*.whl")) + seen = {} + versions = set() + for whl in wheels: + try: + version, platform = validate_wheel(whl) + except (InvalidWheel, ValueError) as e: + raise SystemExit(f"{whl.as_posix()}: {e}") from None + if platform in seen: + raise SystemExit(f"{whl.as_posix()}: second wheel for {platform}, after {seen[platform].as_posix()}") + seen[platform] = whl + versions.add(version) + if set(seen) != EXPECTED_PLATFORMS: + raise SystemExit(f"{wheels_dir.as_posix()}: missing wheels for {sorted(EXPECTED_PLATFORMS - set(seen))}") + if len(versions) != 1: + raise SystemExit(f"{wheels_dir.as_posix()}: wheels have different versions {sorted(versions)}") + return wheels + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit(__doc__) + for whl in validate_dir(Path(sys.argv[1])): + print(f"valid: {whl.as_posix()}") + + +if __name__ == "__main__": + main() From 81c78262c5fc75cb9bc0c2231b9f4b7b21a81062 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 13:09:31 +0000 Subject: [PATCH 10/10] CLI wheels: pin packaging in tools/requirements.txt; smoke test lets uv pick the wheel - packaging is pinned in tools/requirements.txt, where Dependabot tracks it, instead of inline in release-build.yml and the Taskfile. The offline acceptance test uses the vendored packaging. - The release-build smoke test installs from the wheel directory, so uv picks the wheel for the runner instead of a hard-coded linux-amd64 glob. - release-build and the Taskfile use the same uv run options. Co-authored-by: Isaac --- .github/workflows/release-build.yml | 9 ++++++--- Taskfile.yml | 5 +++-- acceptance/cmd/pypi-wheel/output.txt | 2 +- acceptance/cmd/pypi-wheel/script | 5 +++-- tools/requirements.txt | 3 +++ tools/validate_cli_wheels.py | 2 +- 6 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 tools/requirements.txt diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 2978bc349b2..b83123d211d 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -216,6 +216,7 @@ jobs: .github tools/build_cli_wheels.py tools/validate_cli_wheels.py + tools/requirements.txt LICENSE - name: Setup JFrog @@ -239,15 +240,17 @@ jobs: run: uvx twine@6.2.0 check --strict cli-wheels/* - name: Validate wheels - run: uv run -q --no-project --with packaging==25.0 python tools/validate_cli_wheels.py cli-wheels + run: uv run -q -p ">=3.11" --no-project --with-requirements tools/requirements.txt python tools/validate_cli_wheels.py cli-wheels - - name: Smoke test linux-amd64 wheel + # uv picks the wheel for the runner's platform from the directory, as it would + # from PyPI. --no-index keeps it from resolving "databricks" on the index. + - name: Smoke test the wheel for this runner run: | cli_version=$(basename cli-artifact/databricks_cli_*_SHA256SUMS) cli_version=${cli_version#databricks_cli_} cli_version=${cli_version%_SHA256SUMS} uv venv "$RUNNER_TEMP/venv" - uv pip install --python "$RUNNER_TEMP/venv" cli-wheels/*manylinux*x86_64.whl + uv pip install --python "$RUNNER_TEMP/venv" --no-index --find-links cli-wheels databricks "$RUNNER_TEMP/venv/bin/databricks" --version | tee /dev/stderr | grep -F "$cli_version" - name: Upload CLI wheels diff --git a/Taskfile.yml b/Taskfile.yml index 4dc3fa13e0d..2ad60054377 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -266,10 +266,11 @@ tasks: - 'uv run -p ">=3.11" --no-project python tools/validate_codeowners.py' test-cli-wheels: - desc: Unit-test the scripts that package CLI binaries as PyPI wheels and validate them + # release-build.yml runs the scripts themselves on the release binaries. + desc: Unit-test the scripts that build and validate the CLI wheels for PyPI dir: tools cmds: - - 'uv run -p ">=3.11" --no-project --with packaging==25.0 python -m unittest -q test_build_cli_wheels test_validate_cli_wheels' + - 'uv run -q -p ">=3.11" --no-project --with-requirements requirements.txt python -m unittest -q test_build_cli_wheels test_validate_cli_wheels' changelog-preview: desc: Print the CHANGELOG.md section the next release would add from .nextchanges/ diff --git a/acceptance/cmd/pypi-wheel/output.txt b/acceptance/cmd/pypi-wheel/output.txt index 6350374bf61..0403a0ea7a0 100644 --- a/acceptance/cmd/pypi-wheel/output.txt +++ b/acceptance/cmd/pypi-wheel/output.txt @@ -18,7 +18,7 @@ wheels/databricks-1.2.3-py3-none-macosx_13_0_arm64.whl wheels/databricks-1.2.3-py3-none-win_amd64.whl wheels/databricks-1.2.3-py3-none-win_arm64.whl ->>> uv run -q --no-project --with packaging==25.0 python [TESTROOT]/../tools/validate_cli_wheels.py wheels +>>> uv run -q --no-project --with packaging python [TESTROOT]/../tools/validate_cli_wheels.py wheels valid: wheels/databricks-1.2.3-py3-none-macosx_13_0_arm64.whl valid: wheels/databricks-1.2.3-py3-none-macosx_13_0_x86_64.whl valid: wheels/databricks-1.2.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl diff --git a/acceptance/cmd/pypi-wheel/script b/acceptance/cmd/pypi-wheel/script index ddc32543993..df4ad509a0d 100644 --- a/acceptance/cmd/pypi-wheel/script +++ b/acceptance/cmd/pypi-wheel/script @@ -3,8 +3,9 @@ trace python3 make_archives.py $CLI archives trace ls archives trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE -# packaging comes from the vendored packages; the test runs offline. -trace uv run -q --no-project --with packaging==25.0 python $TESTROOT/../tools/validate_cli_wheels.py wheels +# Offline, so packaging comes from the vendored packages rather than the +# tools/requirements.txt pin (they can differ after a Dependabot bump). +trace uv run -q --no-project --with packaging python $TESTROOT/../tools/validate_cli_wheels.py wheels trace python3 show_wheels.py wheels # uv must pick the wheel for this platform: the other wheels hold a placeholder diff --git a/tools/requirements.txt b/tools/requirements.txt new file mode 100644 index 00000000000..f136ca4e458 --- /dev/null +++ b/tools/requirements.txt @@ -0,0 +1,3 @@ +# Python packages for the scripts in tools/ that are not stdlib-only. Pinned here, +# not inline in workflows or the Taskfile, so Dependabot tracks them. +packaging==25.0 diff --git a/tools/validate_cli_wheels.py b/tools/validate_cli_wheels.py index dbc4d57c88e..a3b49d699b7 100644 --- a/tools/validate_cli_wheels.py +++ b/tools/validate_cli_wheels.py @@ -7,7 +7,7 @@ each wheel installs only the CLI binary. Also requires exactly one wheel per supported platform, all with the same version. -Requires `packaging`. Usage: validate_cli_wheels.py +Requires `packaging` (see requirements.txt). Usage: validate_cli_wheels.py """ import base64