Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -309,8 +309,20 @@ tasks:
cmds:
- "uv run --script tools/validate_agents_setup.py"

check-go-sdks:
desc: Check that go.mod's Go SDKs match the releases that .codegen/cli.json and climod/ were generated from
cmds:
- task: test-check-go-sdks
- "uv run --script tools/check_go_sdks.py"

test-check-go-sdks:
desc: Unit-test the Go SDK release check
dir: tools
cmds:
- 'uv run -q -p ">=3.12" --no-project python -m unittest -q test_check_go_sdks'

checks:
desc: Run quick checks (tidy, whitespace, deadcode, changelog, codeowners, lockfiles, agent setup)
desc: Run quick checks (tidy, whitespace, deadcode, changelog, codeowners, lockfiles, agent setup, Go SDKs)
# Sequential: `tidy` rewrites go.mod/go.sum and any future tidy work
# touching more paths should not race with the whitespace scanner.
cmds:
Expand All @@ -321,6 +333,7 @@ tasks:
- task: check-codeowners
- task: check-lockfiles
- task: check-agents-setup
- task: check-go-sdks
- task: test-cli-wheels

install-pythons:
Expand Down
124 changes: 124 additions & 0 deletions tools/check_go_sdks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.12"
# ///
"""Check that the CLI's two Go SDKs and its generated files describe the same release.

The CLI uses both the classic Go SDK (github.com/databricks/databricks-sdk-go) and modules
of the modular Go SDK (github.com/databricks/sdk-go/...). Each SDK release records the
Universe commit it was cut from in .codegen/_last_sha. The daily Go SDK update keeps the
CLI consistent with them, and this check fails when it isn't:

- .codegen/_openapi_sha, which .codegen/cli.json was generated from, must be the classic
Go SDK release's commit.
- climod/.codegen/_last_sha names the sdk-go release that climod/ was generated from. That
must be a published (tagged) sdk-go release, and every sdk-go module in go.mod must be
at its version in that release: the top release in the package's CHANGELOG.md at the
release commit, whose tag must exist, as genkit apply-tag tags it.

The update picks the newest sdk-go release cut from the classic release's commit or an
earlier one. This check can't verify that order, because it needs Universe's history.
Before climod/ is first generated, it has no _last_sha, and only the first check runs.
"""

import json
import os
import subprocess
import sys
import tempfile

CLASSIC_SDK = "github.com/databricks/databricks-sdk-go"
SDK_GO_PREFIX = "github.com/databricks/sdk-go/"
SDK_GO_REPO = "https://github.com/databricks/sdk-go"
RELEASE_MARKER = ".codegen/_last_sha"
CLIMOD_MARKER = "climod/.codegen/_last_sha"
RELEASE_HEADING = "## Release "


def run(*args, cwd=None):
return subprocess.run(args, cwd=cwd, check=True, capture_output=True, text=True).stdout


def release_tag(clone, release, module_path):
"""Return the tag and version of an sdk-go module in a release, as genkit apply-tag does."""
directory = module_path.removeprefix(SDK_GO_PREFIX)
package = json.loads(run("git", "show", f"{release}:{directory}/.package.json", cwd=clone)).get("package", "")
changelog = run("git", "show", f"{release}:{directory}/CHANGELOG.md", cwd=clone)
_, found, heading = changelog.partition(RELEASE_HEADING)
fields = heading.split("\n", 1)[0].split()
if not found or not fields:
raise ValueError(f"{directory}/CHANGELOG.md has no release")
version = fields[0]
return (f"{package}/{version}" if package else version), version


def find_release(clone, universe_sha):
"""Return the newest tagged sdk-go commit whose release marker is universe_sha, or None."""
# A release merges its mirror PR, which updates the marker, before it tags the merged
# commit, so only a tagged commit is a published release.
tagged = set(run("git", "rev-list", "--no-walk", "--tags", cwd=clone).split())
# sdk-go releases from its main branch, whose first-parent history lists the newest
# release first.
for commit in run("git", "rev-list", "--first-parent", "HEAD", cwd=clone).split():
if commit not in tagged:
continue
# Releases from before the marker existed don't have it.
if not run("git", "ls-tree", "--name-only", commit, RELEASE_MARKER, cwd=clone).strip():
continue
if run("git", "show", f"{commit}:{RELEASE_MARKER}", cwd=clone).strip() == universe_sha:
return commit
return None


def check_sdk_go(clone, climod_sha, requires):
"""Return the errors in go.mod's sdk-go modules against the release that climod/ describes."""
release = find_release(clone, climod_sha)
if release is None:
return [f"climod/ was generated from Universe commit {climod_sha}, but no sdk-go release was cut from it."]
errors = []
tags = set(run("git", "tag", cwd=clone).split())
for path, version in sorted(requires.items()):
if not path.startswith(SDK_GO_PREFIX):
continue
tag, want = release_tag(clone, release, path)
if tag not in tags:
errors.append(f"The sdk-go release that climod/ describes hasn't published {path} {want} yet.")
elif want != version:
errors.append(
f"go.mod requires {path} {version}, but the sdk-go release that climod/ describes has {want}. "
f"Pin {want}, or let the Go SDK update move climod/ to a newer sdk-go release."
)
return errors


def main():
errors = []
requires = {r["Path"]: r["Version"] for r in json.loads(run("go", "mod", "edit", "-json")).get("Require", [])}

classic = json.loads(run("go", "mod", "download", "-json", CLASSIC_SDK))
with open(os.path.join(classic["Dir"], RELEASE_MARKER)) as f:
classic_sha = f.read().strip()
with open(".codegen/_openapi_sha") as f:
openapi_sha = f.read().strip()
if openapi_sha != classic_sha:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nice to have] The openapi_sha != classic_sha comparison checks the same invariant as TestConsistentDatabricksSdkVersion in internal/build/sdk_consistency_test.go (lines 20–52), which already runs in CI through test-unit-root. Both read the databricks-sdk-go version from go.mod, obtain its .codegen/_last_sha, and compare it with .codegen/_openapi_sha — the Go test via an HTTP fetch, this script via go mod download. Can one of them be removed, or does having both serve a purpose the existing test doesn't?

errors.append(
f".codegen/_openapi_sha is {openapi_sha}, but {CLASSIC_SDK} {classic['Version']} was released "
f"from Universe commit {classic_sha}. Run the Go SDK update or ./task generate-clijson."
)

if not os.path.exists(CLIMOD_MARKER):
print(f"{CLIMOD_MARKER} doesn't exist yet, so the sdk-go modules aren't checked.")
else:
with open(CLIMOD_MARKER) as f:
climod_sha = f.read().strip()
with tempfile.TemporaryDirectory() as clone:
run("git", "clone", "--quiet", "--filter=blob:none", "--no-checkout", SDK_GO_REPO, clone)
errors += check_sdk_go(clone, climod_sha, requires)

for error in errors:
print(error, file=sys.stderr)
return 1 if errors else 0


if __name__ == "__main__":
sys.exit(main())
75 changes: 75 additions & 0 deletions tools/test_check_go_sdks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import os
import shutil
import subprocess
import tempfile
import unittest

from check_go_sdks import SDK_GO_PREFIX, check_sdk_go, find_release


class SDKGoReleaseTest(unittest.TestCase):
def setUp(self):
self.repo = tempfile.mkdtemp()
self.git("init", "--quiet")
# A release from before the marker existed is tagged but has no marker.
self.git("commit", "--quiet", "--allow-empty", "-m", "Release sdk-go")
self.git("tag", "core/v0.0.1")
self.release("a" * 40, {"auth": "v0.1.0", "core": "v0.1.0"}, ["auth/v0.1.0", "core/v0.1.0"])
# A merged release that was never tagged isn't published.
self.release("b" * 40, {"auth": "v0.2.0"}, [])
# Tagging stopped after auth.
self.release("c" * 40, {"auth": "v0.3.0", "core": "v0.2.0"}, ["auth/v0.3.0"])

def tearDown(self):
shutil.rmtree(self.repo)

def git(self, *args):
env = {
**os.environ,
"GIT_AUTHOR_NAME": "t",
"GIT_AUTHOR_EMAIL": "t@t",
"GIT_COMMITTER_NAME": "t",
"GIT_COMMITTER_EMAIL": "t@t",
}
return subprocess.run(["git", *args], cwd=self.repo, check=True, capture_output=True, text=True, env=env).stdout

def write(self, path, content):
os.makedirs(os.path.dirname(os.path.join(self.repo, path)), exist_ok=True)
with open(os.path.join(self.repo, path), "w") as f:
f.write(content)
self.git("add", path)

def release(self, universe_sha, versions, tags):
self.write(".codegen/_last_sha", universe_sha + "\n")
for package, version in versions.items():
self.write(f"{package}/.package.json", f'{{"package": "{package}"}}')
self.write(f"{package}/CHANGELOG.md", f"# Changelog\n\n## Release {version} (2026-11-18)\n\n- A change.\n")
self.git("commit", "--quiet", "-m", "Release sdk-go")
for tag in tags:
self.git("tag", tag)

def test_find_release(self):
head = self.git("rev-parse", "HEAD").strip()
self.assertEqual(find_release(self.repo, "c" * 40), head)
self.assertIsNotNone(find_release(self.repo, "a" * 40))
self.assertIsNone(find_release(self.repo, "b" * 40))
self.assertIsNone(find_release(self.repo, "d" * 40))

def test_check_sdk_go(self):
auth, core = SDK_GO_PREFIX + "auth", SDK_GO_PREFIX + "core"
classic = "github.com/databricks/databricks-sdk-go"
for name, climod_sha, requires, want_errors in [
("versions match", "a" * 40, {auth: "v0.1.0", core: "v0.1.0", classic: "v1.0.0"}, []),
("version differs", "a" * 40, {auth: "v0.2.0"}, ["has v0.1.0"]),
("module not published", "c" * 40, {auth: "v0.3.0", core: "v0.2.0"}, ["hasn't published " + core]),
("no release", "b" * 40, {auth: "v0.2.0"}, ["no sdk-go release was cut"]),
]:
with self.subTest(name):
errors = check_sdk_go(self.repo, climod_sha, requires)
self.assertEqual(len(errors), len(want_errors), errors)
for error, want in zip(errors, want_errors, strict=True):
self.assertIn(want, error)


if __name__ == "__main__":
unittest.main()
Loading