From 95fe562eba31d4bff7842a831921077b36a6e493 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 17 Nov 2026 13:39:08 +0000 Subject: [PATCH 1/2] Fix wheel compatibility check for serverless job clusters --- .../serverless_job_cluster/databricks.yml | 39 +++++++++++++ .../serverless_job_cluster/out.test.toml | 2 + .../serverless_job_cluster/output.txt | 9 +++ .../trampoline/serverless_job_cluster/script | 1 + bundle/trampoline/python_dbr_warning.go | 2 +- bundle/trampoline/python_dbr_warning_test.go | 56 +++++++++++++++++++ 6 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/trampoline/serverless_job_cluster/databricks.yml create mode 100644 acceptance/bundle/trampoline/serverless_job_cluster/out.test.toml create mode 100644 acceptance/bundle/trampoline/serverless_job_cluster/output.txt create mode 100644 acceptance/bundle/trampoline/serverless_job_cluster/script diff --git a/acceptance/bundle/trampoline/serverless_job_cluster/databricks.yml b/acceptance/bundle/trampoline/serverless_job_cluster/databricks.yml new file mode 100644 index 00000000000..b29ae82dea1 --- /dev/null +++ b/acceptance/bundle/trampoline/serverless_job_cluster/databricks.yml @@ -0,0 +1,39 @@ +bundle: + name: serverless_job_cluster + +resources: + jobs: + serverless_id: + name: serverless-id + job_clusters: + - job_cluster_key: shared + serverless_compute_id: test-compute + environments: + - environment_key: env + spec: + environment_version: "4" + dependencies: + - ./dist/*.whl + tasks: + - task_key: wheel + job_cluster_key: shared + environment_key: env + python_wheel_task: + package_name: my_package + entry_point: main + classic: + name: classic + job_clusters: + - job_cluster_key: shared + new_cluster: + spark_version: 13.3.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + tasks: + - task_key: wheel + job_cluster_key: shared + python_wheel_task: + package_name: my_package + entry_point: main + libraries: + - whl: ./dist/*.whl diff --git a/acceptance/bundle/trampoline/serverless_job_cluster/out.test.toml b/acceptance/bundle/trampoline/serverless_job_cluster/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/trampoline/serverless_job_cluster/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/trampoline/serverless_job_cluster/output.txt b/acceptance/bundle/trampoline/serverless_job_cluster/output.txt new file mode 100644 index 00000000000..b55b9e77a2b --- /dev/null +++ b/acceptance/bundle/trampoline/serverless_job_cluster/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate +Name: serverless_job_cluster +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/serverless_job_cluster/default + +Validation OK! diff --git a/acceptance/bundle/trampoline/serverless_job_cluster/script b/acceptance/bundle/trampoline/serverless_job_cluster/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/trampoline/serverless_job_cluster/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/bundle/trampoline/python_dbr_warning.go b/bundle/trampoline/python_dbr_warning.go index 2b097052dcf..e5ac4a0157b 100644 --- a/bundle/trampoline/python_dbr_warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -62,7 +62,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn if task.JobClusterKey != "" { for _, job := range b.Config.Resources.Jobs { for _, cluster := range job.JobClusters { - if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster.SparkVersion != "" { + if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster != nil && cluster.NewCluster.SparkVersion != "" { if lowerThanExpectedVersion(cluster.NewCluster.SparkVersion) { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, diff --git a/bundle/trampoline/python_dbr_warning_test.go b/bundle/trampoline/python_dbr_warning_test.go index f6ad3ff1cd4..b0835556bf4 100644 --- a/bundle/trampoline/python_dbr_warning_test.go +++ b/bundle/trampoline/python_dbr_warning_test.go @@ -106,6 +106,62 @@ func TestIncompatibleWheelTasksWithJobClusterKey(t *testing.T) { require.ErrorContains(t, diags.Error(), "uses incompatible DBR version 12.2.x-scala2.12") } +func TestWheelTasksWithServerlessJobCluster(t *testing.T) { + for _, tc := range []struct { + name string + cluster jobs.JobCluster + }{ + { + name: "serverless_compute", + cluster: jobs.JobCluster{ + ServerlessCompute: &jobs.JobClusterServerlessCompute{}, + }, + }, + { + name: "serverless_compute_policy", + cluster: jobs.JobCluster{ + ServerlessCompute: &jobs.JobClusterServerlessCompute{PolicyId: "test-policy"}, + }, + }, + { + name: "serverless_compute_id", + cluster: jobs.JobCluster{ + ServerlessComputeId: "test-compute", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cluster := tc.cluster + cluster.JobClusterKey = "shared" + job := &resources.Job{} + job.JobClusters = []jobs.JobCluster{cluster} + job.Tasks = []jobs.Task{{ + TaskKey: "wheel", + JobClusterKey: cluster.JobClusterKey, + PythonWheelTask: &jobs.PythonWheelTask{PackageName: "my_package", EntryPoint: "main"}, + EnvironmentKey: "env", + }} + job.Environments = []jobs.JobEnvironment{{ + EnvironmentKey: "env", + Spec: &compute.Environment{ + EnvironmentVersion: "4", + Dependencies: []string{"./dist/test.whl"}, + }, + }} + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"serverless": job}, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, WrapperWarning()) + require.Empty(t, diags) + }) + } +} + func TestIncompatibleWheelTasksWithExistingClusterId(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ From c4c2e32fb1d0f6a571e06aee3b148c1e924a6698 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 17 Nov 2026 13:39:44 +0000 Subject: [PATCH 2/2] Add changelog entry for serverless wheel compatibility fix --- .nextchanges/bundles/serverless-wheel-compatibility.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/serverless-wheel-compatibility.md diff --git a/.nextchanges/bundles/serverless-wheel-compatibility.md b/.nextchanges/bundles/serverless-wheel-compatibility.md new file mode 100644 index 00000000000..c559971a3a4 --- /dev/null +++ b/.nextchanges/bundles/serverless-wheel-compatibility.md @@ -0,0 +1 @@ +* Fix a panic when validating or deploying jobs with local libraries and serverless job clusters. ([#7725](https://github.com/databricks/cli/pull/7725))