diff --git a/.nextchanges/bundles/serverless-wheel-compatibility.md b/.nextchanges/bundles/serverless-wheel-compatibility.md new file mode 100644 index 00000000000..c559971a3a4 --- /dev/null +++ b/.nextchanges/bundles/serverless-wheel-compatibility.md @@ -0,0 +1 @@ +* Fix a panic when validating or deploying jobs with local libraries and serverless job clusters. ([#7725](https://github.com/databricks/cli/pull/7725)) diff --git a/acceptance/bundle/trampoline/serverless_job_cluster/databricks.yml b/acceptance/bundle/trampoline/serverless_job_cluster/databricks.yml new file mode 100644 index 00000000000..b29ae82dea1 --- /dev/null +++ b/acceptance/bundle/trampoline/serverless_job_cluster/databricks.yml @@ -0,0 +1,39 @@ +bundle: + name: serverless_job_cluster + +resources: + jobs: + serverless_id: + name: serverless-id + job_clusters: + - job_cluster_key: shared + serverless_compute_id: test-compute + environments: + - environment_key: env + spec: + environment_version: "4" + dependencies: + - ./dist/*.whl + tasks: + - task_key: wheel + job_cluster_key: shared + environment_key: env + python_wheel_task: + package_name: my_package + entry_point: main + classic: + name: classic + job_clusters: + - job_cluster_key: shared + new_cluster: + spark_version: 13.3.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + tasks: + - task_key: wheel + job_cluster_key: shared + python_wheel_task: + package_name: my_package + entry_point: main + libraries: + - whl: ./dist/*.whl diff --git a/acceptance/bundle/trampoline/serverless_job_cluster/out.test.toml b/acceptance/bundle/trampoline/serverless_job_cluster/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/trampoline/serverless_job_cluster/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/trampoline/serverless_job_cluster/output.txt b/acceptance/bundle/trampoline/serverless_job_cluster/output.txt new file mode 100644 index 00000000000..b55b9e77a2b --- /dev/null +++ b/acceptance/bundle/trampoline/serverless_job_cluster/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate +Name: serverless_job_cluster +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/serverless_job_cluster/default + +Validation OK! diff --git a/acceptance/bundle/trampoline/serverless_job_cluster/script b/acceptance/bundle/trampoline/serverless_job_cluster/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/trampoline/serverless_job_cluster/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/bundle/trampoline/python_dbr_warning.go b/bundle/trampoline/python_dbr_warning.go index 2b097052dcf..e5ac4a0157b 100644 --- a/bundle/trampoline/python_dbr_warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -62,7 +62,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn if task.JobClusterKey != "" { for _, job := range b.Config.Resources.Jobs { for _, cluster := range job.JobClusters { - if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster.SparkVersion != "" { + if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster != nil && cluster.NewCluster.SparkVersion != "" { if lowerThanExpectedVersion(cluster.NewCluster.SparkVersion) { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, diff --git a/bundle/trampoline/python_dbr_warning_test.go b/bundle/trampoline/python_dbr_warning_test.go index f6ad3ff1cd4..b0835556bf4 100644 --- a/bundle/trampoline/python_dbr_warning_test.go +++ b/bundle/trampoline/python_dbr_warning_test.go @@ -106,6 +106,62 @@ func TestIncompatibleWheelTasksWithJobClusterKey(t *testing.T) { require.ErrorContains(t, diags.Error(), "uses incompatible DBR version 12.2.x-scala2.12") } +func TestWheelTasksWithServerlessJobCluster(t *testing.T) { + for _, tc := range []struct { + name string + cluster jobs.JobCluster + }{ + { + name: "serverless_compute", + cluster: jobs.JobCluster{ + ServerlessCompute: &jobs.JobClusterServerlessCompute{}, + }, + }, + { + name: "serverless_compute_policy", + cluster: jobs.JobCluster{ + ServerlessCompute: &jobs.JobClusterServerlessCompute{PolicyId: "test-policy"}, + }, + }, + { + name: "serverless_compute_id", + cluster: jobs.JobCluster{ + ServerlessComputeId: "test-compute", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cluster := tc.cluster + cluster.JobClusterKey = "shared" + job := &resources.Job{} + job.JobClusters = []jobs.JobCluster{cluster} + job.Tasks = []jobs.Task{{ + TaskKey: "wheel", + JobClusterKey: cluster.JobClusterKey, + PythonWheelTask: &jobs.PythonWheelTask{PackageName: "my_package", EntryPoint: "main"}, + EnvironmentKey: "env", + }} + job.Environments = []jobs.JobEnvironment{{ + EnvironmentKey: "env", + Spec: &compute.Environment{ + EnvironmentVersion: "4", + Dependencies: []string{"./dist/test.whl"}, + }, + }} + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{"serverless": job}, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, WrapperWarning()) + require.Empty(t, diags) + }) + } +} + func TestIncompatibleWheelTasksWithExistingClusterId(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{