From dd604067386c19f47e1c04396f160596400d155e Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 13:28:05 +0000 Subject: [PATCH 1/2] acc: add tests for unquoted numeric YAML scalars in string fields Cover fields that hold ids, versions and tags, both as literals and via ${var.x}. Outputs record the current lossy conversion (leading zeros, trailing zeros, exponents, hex/octal, interpolated floats). Co-authored-by: Isaac --- .../variables/numeric_defaults/databricks.yml | 56 ++++++++++ .../variables/numeric_defaults/out.test.toml | 2 + .../variables/numeric_defaults/output.txt | 28 +++++ .../variables/numeric_defaults/script | 6 ++ .../dashboard.lvdash.json | 1 + .../yaml_numeric_strings/databricks.yml | 100 +++++++++++++++++ .../yaml_numeric_strings/out.test.toml | 2 + .../validate/yaml_numeric_strings/output.txt | 101 ++++++++++++++++++ .../validate/yaml_numeric_strings/script | 9 ++ 9 files changed, 305 insertions(+) create mode 100644 acceptance/bundle/validate/variables/numeric_defaults/databricks.yml create mode 100644 acceptance/bundle/validate/variables/numeric_defaults/out.test.toml create mode 100644 acceptance/bundle/validate/variables/numeric_defaults/output.txt create mode 100644 acceptance/bundle/validate/variables/numeric_defaults/script create mode 100644 acceptance/bundle/validate/yaml_numeric_strings/dashboard.lvdash.json create mode 100644 acceptance/bundle/validate/yaml_numeric_strings/databricks.yml create mode 100644 acceptance/bundle/validate/yaml_numeric_strings/out.test.toml create mode 100644 acceptance/bundle/validate/yaml_numeric_strings/output.txt create mode 100644 acceptance/bundle/validate/yaml_numeric_strings/script diff --git a/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml b/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml new file mode 100644 index 00000000000..9573f8b84a5 --- /dev/null +++ b/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml @@ -0,0 +1,56 @@ +# Variable defaults that YAML resolves to a number, referenced from string fields. +bundle: + name: numeric-defaults + +variables: + # YAML resolves this to a float: with a leading "0" it is not a valid octal int (has 8, 9). + leading_zero: + default: 0012345678901234 + # YAML resolves this to an int (valid octal digits only); the loader parses it as decimal 42. + leading_zero_int: + default: 0042 + exponent: + default: 12345678901E0042 + version: + default: 3.10 + quoted: + default: "0012345678901234" + # Overridden in the target below, see targets.dev.variables. + target_override: + default: "0012345678901234" + # Overridden with --var and BUNDLE_VAR_ in the script. + from_flag: + default: placeholder + from_env: + default: placeholder + +resources: + jobs: + job: + name: job + tasks: + - task_key: t + new_cluster: + policy_id: ${var.leading_zero} + spark_version: 13.3.x-scala2.12 + num_workers: 1 + notebook_task: + notebook_path: /Workspace/notebook + base_parameters: + leading_zero: ${var.leading_zero} + exponent: ${var.exponent} + version: ${var.version} + quoted: ${var.quoted} + target_override: ${var.target_override} + from_flag: ${var.from_flag} + from_env: ${var.from_env} + leading_zero_int: ${var.leading_zero_int} + interpolated: id-${var.leading_zero} + interpolated_int: id-${var.leading_zero_int} + interpolated_version: v${var.version} + +targets: + dev: + default: true + variables: + target_override: 0012345678901234 diff --git a/acceptance/bundle/validate/variables/numeric_defaults/out.test.toml b/acceptance/bundle/validate/variables/numeric_defaults/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/variables/numeric_defaults/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/variables/numeric_defaults/output.txt b/acceptance/bundle/validate/variables/numeric_defaults/output.txt new file mode 100644 index 00000000000..4764e51cbf8 --- /dev/null +++ b/acceptance/bundle/validate/variables/numeric_defaults/output.txt @@ -0,0 +1,28 @@ + +>>> [CLI] bundle validate -o json --var from_flag=0012345678901234 +{ + "variables": { + "exponent": 1.2345678901E+52, + "from_env": "0012345678901234", + "from_flag": "0012345678901234", + "leading_zero": 12345678901234, + "leading_zero_int": 42, + "quoted": "0012345678901234", + "target_override": 12345678901234, + "version": 3.1 + }, + "policy_id": "12345678901234", + "base_parameters": { + "exponent": "12345678901000000000000000000000000000000000000000000", + "from_env": "0012345678901234", + "from_flag": "0012345678901234", + "interpolated": "id-1.2345678901234e+13", + "interpolated_int": "id-42", + "interpolated_version": "v3.1", + "leading_zero": "12345678901234", + "leading_zero_int": "42", + "quoted": "0012345678901234", + "target_override": "12345678901234", + "version": "3.1" + } +} diff --git a/acceptance/bundle/validate/variables/numeric_defaults/script b/acceptance/bundle/validate/variables/numeric_defaults/script new file mode 100644 index 00000000000..2c20b6dc24c --- /dev/null +++ b/acceptance/bundle/validate/variables/numeric_defaults/script @@ -0,0 +1,6 @@ +export BUNDLE_VAR_from_env=0012345678901234 +trace $CLI bundle validate -o json --var from_flag=0012345678901234 | jq '{ + variables: .variables | map_values(.value), + policy_id: .resources.jobs.job.tasks[0].new_cluster.policy_id, + base_parameters: .resources.jobs.job.tasks[0].notebook_task.base_parameters +}' diff --git a/acceptance/bundle/validate/yaml_numeric_strings/dashboard.lvdash.json b/acceptance/bundle/validate/yaml_numeric_strings/dashboard.lvdash.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/dashboard.lvdash.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml b/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml new file mode 100644 index 00000000000..986b3676ece --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml @@ -0,0 +1,100 @@ +# Unquoted scalars that YAML resolves to a number, bool or timestamp, used in +# string-typed fields. Each value is written twice: unquoted and quoted ("q_"). +# The quoted value is kept verbatim; the unquoted one is converted from the +# resolved number and may not match the source text. +bundle: + name: yaml-numeric-strings + +resources: + jobs: + job: + name: 007 + tags: + leading_zero: 0042 + q_leading_zero: "0042" + trailing_zero: 1.10 + q_trailing_zero: "1.10" + float_int: 1.0 + q_float_int: "1.0" + exponent: 1e3 + q_exponent: "1e3" + hex: 0x1F + q_hex: "0x1F" + octal: 0o17 + q_octal: "0o17" + underscore: 1_000 + q_underscore: "1_000" + plus_sign: +12 + q_plus_sign: "+12" + leading_dot: .5 + q_leading_dot: ".5" + big_int: 123456789012345678901234567890 + q_big_int: "123456789012345678901234567890" + inf: .inf + q_inf: ".inf" + nan: .NaN + q_nan: ".NaN" + bool_title: True + q_bool_title: "True" + date: 2024-01-01 + q_date: "2024-01-01" + parameters: + - name: leading_zero + default: 01 + - name: version + default: 3.10 + git_source: + git_url: https://github.com/databricks/cli + git_provider: gitHub + # 7-char short SHA with a single "e" and digits only around it + git_commit: 1234e56 + tasks: + - task_key: cluster_policy + new_cluster: + # Cluster policy ids are 16 hex characters. + policy_id: 0012345678901234 + spark_version: 13.30 + num_workers: 1 + spark_conf: + spark.sql.shuffle.partitions: 0200 + spark_env_vars: + PYTHON_VERSION: 3.10 + custom_tags: + cost_center: 0042 + notebook_task: + notebook_path: /Workspace/notebook + base_parameters: + leading_zero: 0042 + q_leading_zero: "0042" + - task_key: cluster_policy_exponent + new_cluster: + policy_id: 12345678901E0042 + spark_version: 13.3.x-scala2.12 + num_workers: 1 + notebook_task: + notebook_path: /Workspace/notebook + - task_key: existing_cluster + existing_cluster_id: 0123 + sql_task: + # SQL warehouse ids are 16 hex characters. + warehouse_id: 0123456789012345 + query: + query_id: 1e5 + + pipelines: + pipeline: + name: pipeline + catalog: 0001 + schema: 2024 + configuration: + leading_zero: 0042 + version: 1.10 + clusters: + - label: default + policy_id: 0012345678901234 + + dashboards: + dashboard: + display_name: dashboard + warehouse_id: 0123456789012345 + file_path: dashboard.lvdash.json diff --git a/acceptance/bundle/validate/yaml_numeric_strings/out.test.toml b/acceptance/bundle/validate/yaml_numeric_strings/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_numeric_strings/output.txt b/acceptance/bundle/validate/yaml_numeric_strings/output.txt new file mode 100644 index 00000000000..60ca5c73bf5 --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/output.txt @@ -0,0 +1,101 @@ + +>>> [CLI] bundle validate -o json +{ + "job_name": "7", + "job_tags": { + "big_int": "123456789012345680000000000000", + "bool_title": "true", + "date": "2024-01-01", + "exponent": "1000", + "float_int": "1", + "hex": "31", + "inf": "+Inf", + "leading_dot": "0.5", + "leading_zero": "42", + "nan": "NaN", + "octal": "15", + "plus_sign": "12", + "q_big_int": "123456789012345678901234567890", + "q_bool_title": "True", + "q_date": "2024-01-01", + "q_exponent": "1e3", + "q_float_int": "1.0", + "q_hex": "0x1F", + "q_inf": ".inf", + "q_leading_dot": ".5", + "q_leading_zero": "0042", + "q_nan": ".NaN", + "q_octal": "0o17", + "q_plus_sign": "+12", + "q_trailing_zero": "1.10", + "q_underscore": "1_000", + "trailing_zero": "1.1", + "underscore": "1000" + }, + "job_parameters": [ + { + "default": "1", + "name": "leading_zero" + }, + { + "default": "3.1", + "name": "version" + } + ], + "git_commit": "123400000000000000000000000000000000000000000000000000000000", + "tasks": [ + { + "new_cluster": { + "custom_tags": { + "cost_center": "42" + }, + "policy_id": "12345678901234", + "spark_conf": { + "spark.sql.shuffle.partitions": "200" + }, + "spark_env_vars": { + "PYTHON_VERSION": "3.1" + }, + "spark_version": "13.3" + }, + "notebook_task": { + "base_parameters": { + "leading_zero": "42", + "q_leading_zero": "0042" + }, + "source": "GIT" + }, + "task_key": "cluster_policy" + }, + { + "new_cluster": { + "policy_id": "12345678901000000000000000000000000000000000000000000", + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "source": "GIT" + }, + "task_key": "cluster_policy_exponent" + }, + { + "existing_cluster_id": "123", + "sql_task": { + "query": { + "query_id": "100000" + }, + "warehouse_id": "123456789012345" + }, + "task_key": "existing_cluster" + } + ], + "pipeline": { + "catalog": "1", + "schema": "2024", + "configuration": { + "leading_zero": "42", + "version": "1.1" + }, + "policy_id": "12345678901234" + }, + "dashboard_warehouse_id": "123456789012345" +} diff --git a/acceptance/bundle/validate/yaml_numeric_strings/script b/acceptance/bundle/validate/yaml_numeric_strings/script new file mode 100644 index 00000000000..613a31a4cc2 --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/script @@ -0,0 +1,9 @@ +trace $CLI bundle validate -o json | jq '.resources | { + job_name: .jobs.job.name, + job_tags: .jobs.job.tags, + job_parameters: .jobs.job.parameters, + git_commit: .jobs.job.git_source.git_commit, + tasks: [.jobs.job.tasks[] | del(.notebook_task.notebook_path, .new_cluster.num_workers)], + pipeline: .pipelines.pipeline | {catalog, schema, configuration, policy_id: .clusters[0].policy_id}, + dashboard_warehouse_id: .dashboards.dashboard.warehouse_id +}' From 58980cea9ab2b6d55847649cab790f89807b501e Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 15:36:44 +0000 Subject: [PATCH 2/2] acc: check that a numeric variable still parses in an int field Co-authored-by: Isaac --- .../bundle/validate/variables/numeric_defaults/databricks.yml | 4 ++++ .../bundle/validate/variables/numeric_defaults/output.txt | 2 ++ acceptance/bundle/validate/variables/numeric_defaults/script | 1 + 3 files changed, 7 insertions(+) diff --git a/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml b/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml index 9573f8b84a5..284f1dd87be 100644 --- a/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml +++ b/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml @@ -15,6 +15,9 @@ variables: default: 3.10 quoted: default: "0012345678901234" + # Used in an int field, which must get the parsed number. + concurrency: + default: 010 # Overridden in the target below, see targets.dev.variables. target_override: default: "0012345678901234" @@ -28,6 +31,7 @@ resources: jobs: job: name: job + max_concurrent_runs: ${var.concurrency} tasks: - task_key: t new_cluster: diff --git a/acceptance/bundle/validate/variables/numeric_defaults/output.txt b/acceptance/bundle/validate/variables/numeric_defaults/output.txt index 4764e51cbf8..a203b254b1b 100644 --- a/acceptance/bundle/validate/variables/numeric_defaults/output.txt +++ b/acceptance/bundle/validate/variables/numeric_defaults/output.txt @@ -2,6 +2,7 @@ >>> [CLI] bundle validate -o json --var from_flag=0012345678901234 { "variables": { + "concurrency": 10, "exponent": 1.2345678901E+52, "from_env": "0012345678901234", "from_flag": "0012345678901234", @@ -11,6 +12,7 @@ "target_override": 12345678901234, "version": 3.1 }, + "max_concurrent_runs": 10, "policy_id": "12345678901234", "base_parameters": { "exponent": "12345678901000000000000000000000000000000000000000000", diff --git a/acceptance/bundle/validate/variables/numeric_defaults/script b/acceptance/bundle/validate/variables/numeric_defaults/script index 2c20b6dc24c..27e5ebdf670 100644 --- a/acceptance/bundle/validate/variables/numeric_defaults/script +++ b/acceptance/bundle/validate/variables/numeric_defaults/script @@ -1,6 +1,7 @@ export BUNDLE_VAR_from_env=0012345678901234 trace $CLI bundle validate -o json --var from_flag=0012345678901234 | jq '{ variables: .variables | map_values(.value), + max_concurrent_runs: .resources.jobs.job.max_concurrent_runs, policy_id: .resources.jobs.job.tasks[0].new_cluster.policy_id, base_parameters: .resources.jobs.job.tasks[0].notebook_task.base_parameters }'