diff --git a/.nextchanges/bundles/pipelines-dry-run-job-key.md b/.nextchanges/bundles/pipelines-dry-run-job-key.md new file mode 100644 index 00000000000..92200055c66 --- /dev/null +++ b/.nextchanges/bundles/pipelines-dry-run-job-key.md @@ -0,0 +1 @@ +* Fix `pipelines dry-run` starting a real run when given a job KEY, and a crash in its shell completion. ([#7672](https://github.com/databricks/cli/pull/7672)) diff --git a/acceptance/pipelines/dry-run/job-key/databricks.yml b/acceptance/pipelines/dry-run/job-key/databricks.yml new file mode 100644 index 00000000000..a63fb292dd9 --- /dev/null +++ b/acceptance/pipelines/dry-run/job-key/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: test-pipeline-dry-run-job-key + +resources: + pipelines: + my_pipeline: + name: test-pipeline + jobs: + my_job: + name: My Job + tasks: + - task_key: main + pipeline_task: + pipeline_id: ${resources.pipelines.my_pipeline.id} diff --git a/acceptance/pipelines/dry-run/job-key/out.test.toml b/acceptance/pipelines/dry-run/job-key/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/pipelines/dry-run/job-key/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/pipelines/dry-run/job-key/output.txt b/acceptance/pipelines/dry-run/job-key/output.txt new file mode 100644 index 00000000000..3e83df8b48c --- /dev/null +++ b/acceptance/pipelines/dry-run/job-key/output.txt @@ -0,0 +1,47 @@ + +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-dry-run-job-key/default/files... +Created jobs.my_job +Created pipelines.my_pipeline +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +View your job my_job here: [DATABRICKS_URL]/jobs/[JOB_ID]?w=[WORKSPACE_ID] +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID][0]?w=[WORKSPACE_ID] + +=== Dry running a job KEY + +>>> [CLI] pipelines dry-run my_job +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Error: resource my_job is not a pipeline + +Exit code: 1 + +=== Completion only offers pipelines + +>>> [CLI] __complete pipelines dry-run , +my_pipeline +:4 +Completion ended with directive: ShellCompDirectiveNoFileComp + +=== Dry running the pipeline KEY + +>>> [CLI] pipelines dry-run my_pipeline +Recommendation: This command runs the last deployed version of the code + +If you've made local changes, run 'databricks pipelines deploy' first to ensure they are included. + +Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID][0]/updates/[UUID][1] + +Update ID: [UUID][1] + +=== Only the pipeline update is requested, with validate_only set +{ + "method": "POST", + "path": "/api/2.0/pipelines/[UUID][0]/updates", + "body": { + "validate_only": true + } +} diff --git a/acceptance/pipelines/dry-run/job-key/script b/acceptance/pipelines/dry-run/job-key/script new file mode 100644 index 00000000000..f3131cdc4d5 --- /dev/null +++ b/acceptance/pipelines/dry-run/job-key/script @@ -0,0 +1,19 @@ +trace $CLI pipelines deploy +rm "$OUT_REQUESTS" + +# A dry run must never execute anything. A job has no dry-run mode. +title "Dry running a job KEY" +errcode trace $CLI pipelines dry-run my_job + +title "Completion only offers pipelines" +errcode trace $CLI __complete pipelines dry-run , &> complete.txt +# Drop the machine-specific stack trace if completion panics. +sed '/^Stack Trace:/,$d' complete.txt +rm complete.txt + +# Make sure that a dry run of the pipeline itself still works. +title "Dry running the pipeline KEY" +trace $CLI pipelines dry-run my_pipeline + +title "Only the pipeline update is requested, with validate_only set" +print_requests.py //jobs/run-now //pipelines diff --git a/acceptance/pipelines/dry-run/job-key/test.toml b/acceptance/pipelines/dry-run/job-key/test.toml new file mode 100644 index 00000000000..14a01ee8761 --- /dev/null +++ b/acceptance/pipelines/dry-run/job-key/test.toml @@ -0,0 +1,14 @@ +RecordRequests = true +Ignore = [".databricks"] + +[[Repls]] +Old = '/runs/\d+' +New = '/runs/[RUN_ID]' + +[[Repls]] +Old = '/jobs/\d+' +New = '/jobs/[JOB_ID]' + +[[Repls]] +Old = '"job_id": \d+' +New = '"job_id": [JOB_ID]' diff --git a/cmd/pipelines/dry_run.go b/cmd/pipelines/dry_run.go index ec74f7f7b0f..4841ac10c99 100644 --- a/cmd/pipelines/dry_run.go +++ b/cmd/pipelines/dry_run.go @@ -25,6 +25,7 @@ func dryRunCommand() *cobra.Command { Long: `Validates correctness of the pipeline's graph, identified by KEY. Does not materialize or publish any datasets. KEY is the unique name of the pipeline to dry run, as defined in its YAML file. If there is only one pipeline in the project, KEY is optional and the pipeline will be auto-selected.`, + Args: root.MaximumNArgs(1), } var noWait bool @@ -46,11 +47,16 @@ If there is only one pipeline in the project, KEY is optional and the pipeline w suggestPipelineDeploy(ctx, cmd) - key, _, err := resolveRunArgument(ctx, b, args) + key, err := resolvePipelineArgument(ctx, b, args) if err != nil { return err } + // The runner accepts any runnable resource, and a job has no validate-only mode. + if _, err := resolvePipelineIdFromKey(ctx, b, key); err != nil { + return err + } + runner, err := keyToRunner(b, key) if err != nil { return err @@ -103,6 +109,9 @@ If there is only one pipeline in the project, KEY is optional and the pipeline w } cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + b := root.MustConfigureBundle(cmd) if logdiag.HasError(cmd.Context()) { return nil, cobra.ShellCompDirectiveError @@ -115,16 +124,10 @@ If there is only one pipeline in the project, KEY is optional and the pipeline w } if len(args) == 0 { - completions := resources.Completions(b, run.IsRunnable) + completions := resources.Completions(b, isPipeline) return slices.Collect(maps.Keys(completions)), cobra.ShellCompDirectiveNoFileComp - } else { - // If we know the resource to run, we can complete additional positional arguments. - runner, err := keyToRunner(b, args[0]) - if err != nil { - return nil, cobra.ShellCompDirectiveError - } - return runner.CompleteArgs(args[1:], toComplete) } + return nil, cobra.ShellCompDirectiveNoFileComp } return cmd