Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/serverless-wheel-compatibility.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Fix a panic when validating or deploying jobs with local libraries and serverless job clusters. ([#7725](https://github.com/databricks/cli/pull/7725))
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
bundle:
name: serverless_job_cluster

resources:
jobs:
serverless_id:
name: serverless-id
job_clusters:
- job_cluster_key: shared
serverless_compute_id: test-compute
environments:
- environment_key: env
spec:
environment_version: "4"
dependencies:
- ./dist/*.whl
tasks:
- task_key: wheel
job_cluster_key: shared
environment_key: env
python_wheel_task:
package_name: my_package
entry_point: main
classic:
name: classic
job_clusters:
- job_cluster_key: shared
new_cluster:
spark_version: 13.3.x-scala2.12
node_type_id: i3.xlarge
num_workers: 1
tasks:
- task_key: wheel
job_cluster_key: shared
python_wheel_task:
package_name: my_package
entry_point: main
libraries:
- whl: ./dist/*.whl
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@

>>> [CLI] bundle validate
Name: serverless_job_cluster
Target: default
Workspace:
User: [USERNAME]
Path: /Workspace/Users/[USERNAME]/.bundle/serverless_job_cluster/default

Validation OK!
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
trace $CLI bundle validate
2 changes: 1 addition & 1 deletion bundle/trampoline/python_dbr_warning.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn
if task.JobClusterKey != "" {
for _, job := range b.Config.Resources.Jobs {
for _, cluster := range job.JobClusters {
if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster.SparkVersion != "" {
if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster != nil && cluster.NewCluster.SparkVersion != "" {
if lowerThanExpectedVersion(cluster.NewCluster.SparkVersion) {
diags = append(diags, diag.Diagnostic{
Severity: diag.Error,
Expand Down
56 changes: 56 additions & 0 deletions bundle/trampoline/python_dbr_warning_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,62 @@ func TestIncompatibleWheelTasksWithJobClusterKey(t *testing.T) {
require.ErrorContains(t, diags.Error(), "uses incompatible DBR version 12.2.x-scala2.12")
}

func TestWheelTasksWithServerlessJobCluster(t *testing.T) {
for _, tc := range []struct {
name string
cluster jobs.JobCluster
}{
{
name: "serverless_compute",
cluster: jobs.JobCluster{
ServerlessCompute: &jobs.JobClusterServerlessCompute{},
},
},
{
name: "serverless_compute_policy",
cluster: jobs.JobCluster{
ServerlessCompute: &jobs.JobClusterServerlessCompute{PolicyId: "test-policy"},
},
},
{
name: "serverless_compute_id",
cluster: jobs.JobCluster{
ServerlessComputeId: "test-compute",
},
},
} {
t.Run(tc.name, func(t *testing.T) {
cluster := tc.cluster
cluster.JobClusterKey = "shared"
job := &resources.Job{}
job.JobClusters = []jobs.JobCluster{cluster}
job.Tasks = []jobs.Task{{
TaskKey: "wheel",
JobClusterKey: cluster.JobClusterKey,
PythonWheelTask: &jobs.PythonWheelTask{PackageName: "my_package", EntryPoint: "main"},
EnvironmentKey: "env",
}}
job.Environments = []jobs.JobEnvironment{{
EnvironmentKey: "env",
Spec: &compute.Environment{
EnvironmentVersion: "4",
Dependencies: []string{"./dist/test.whl"},
},
}}
b := &bundle.Bundle{
Config: config.Root{
Resources: config.Resources{
Jobs: map[string]*resources.Job{"serverless": job},
},
},
}

diags := bundle.Apply(t.Context(), b, WrapperWarning())
require.Empty(t, diags)
})
}
}

func TestIncompatibleWheelTasksWithExistingClusterId(t *testing.T) {
b := &bundle.Bundle{
Config: config.Root{
Expand Down
Loading