diff --git a/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml b/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml new file mode 100644 index 00000000000..284f1dd87be --- /dev/null +++ b/acceptance/bundle/validate/variables/numeric_defaults/databricks.yml @@ -0,0 +1,60 @@ +# Variable defaults that YAML resolves to a number, referenced from string fields. +bundle: + name: numeric-defaults + +variables: + # YAML resolves this to a float: with a leading "0" it is not a valid octal int (has 8, 9). + leading_zero: + default: 0012345678901234 + # YAML resolves this to an int (valid octal digits only); the loader parses it as decimal 42. + leading_zero_int: + default: 0042 + exponent: + default: 12345678901E0042 + version: + default: 3.10 + quoted: + default: "0012345678901234" + # Used in an int field, which must get the parsed number. + concurrency: + default: 010 + # Overridden in the target below, see targets.dev.variables. + target_override: + default: "0012345678901234" + # Overridden with --var and BUNDLE_VAR_ in the script. + from_flag: + default: placeholder + from_env: + default: placeholder + +resources: + jobs: + job: + name: job + max_concurrent_runs: ${var.concurrency} + tasks: + - task_key: t + new_cluster: + policy_id: ${var.leading_zero} + spark_version: 13.3.x-scala2.12 + num_workers: 1 + notebook_task: + notebook_path: /Workspace/notebook + base_parameters: + leading_zero: ${var.leading_zero} + exponent: ${var.exponent} + version: ${var.version} + quoted: ${var.quoted} + target_override: ${var.target_override} + from_flag: ${var.from_flag} + from_env: ${var.from_env} + leading_zero_int: ${var.leading_zero_int} + interpolated: id-${var.leading_zero} + interpolated_int: id-${var.leading_zero_int} + interpolated_version: v${var.version} + +targets: + dev: + default: true + variables: + target_override: 0012345678901234 diff --git a/acceptance/bundle/validate/variables/numeric_defaults/out.test.toml b/acceptance/bundle/validate/variables/numeric_defaults/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/variables/numeric_defaults/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/variables/numeric_defaults/output.txt b/acceptance/bundle/validate/variables/numeric_defaults/output.txt new file mode 100644 index 00000000000..a203b254b1b --- /dev/null +++ b/acceptance/bundle/validate/variables/numeric_defaults/output.txt @@ -0,0 +1,30 @@ + +>>> [CLI] bundle validate -o json --var from_flag=0012345678901234 +{ + "variables": { + "concurrency": 10, + "exponent": 1.2345678901E+52, + "from_env": "0012345678901234", + "from_flag": "0012345678901234", + "leading_zero": 12345678901234, + "leading_zero_int": 42, + "quoted": "0012345678901234", + "target_override": 12345678901234, + "version": 3.1 + }, + "max_concurrent_runs": 10, + "policy_id": "12345678901234", + "base_parameters": { + "exponent": "12345678901000000000000000000000000000000000000000000", + "from_env": "0012345678901234", + "from_flag": "0012345678901234", + "interpolated": "id-1.2345678901234e+13", + "interpolated_int": "id-42", + "interpolated_version": "v3.1", + "leading_zero": "12345678901234", + "leading_zero_int": "42", + "quoted": "0012345678901234", + "target_override": "12345678901234", + "version": "3.1" + } +} diff --git a/acceptance/bundle/validate/variables/numeric_defaults/script b/acceptance/bundle/validate/variables/numeric_defaults/script new file mode 100644 index 00000000000..27e5ebdf670 --- /dev/null +++ b/acceptance/bundle/validate/variables/numeric_defaults/script @@ -0,0 +1,7 @@ +export BUNDLE_VAR_from_env=0012345678901234 +trace $CLI bundle validate -o json --var from_flag=0012345678901234 | jq '{ + variables: .variables | map_values(.value), + max_concurrent_runs: .resources.jobs.job.max_concurrent_runs, + policy_id: .resources.jobs.job.tasks[0].new_cluster.policy_id, + base_parameters: .resources.jobs.job.tasks[0].notebook_task.base_parameters +}' diff --git a/acceptance/bundle/validate/yaml_numeric_strings/dashboard.lvdash.json b/acceptance/bundle/validate/yaml_numeric_strings/dashboard.lvdash.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/dashboard.lvdash.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml b/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml new file mode 100644 index 00000000000..986b3676ece --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml @@ -0,0 +1,100 @@ +# Unquoted scalars that YAML resolves to a number, bool or timestamp, used in +# string-typed fields. Each value is written twice: unquoted and quoted ("q_"). +# The quoted value is kept verbatim; the unquoted one is converted from the +# resolved number and may not match the source text. +bundle: + name: yaml-numeric-strings + +resources: + jobs: + job: + name: 007 + tags: + leading_zero: 0042 + q_leading_zero: "0042" + trailing_zero: 1.10 + q_trailing_zero: "1.10" + float_int: 1.0 + q_float_int: "1.0" + exponent: 1e3 + q_exponent: "1e3" + hex: 0x1F + q_hex: "0x1F" + octal: 0o17 + q_octal: "0o17" + underscore: 1_000 + q_underscore: "1_000" + plus_sign: +12 + q_plus_sign: "+12" + leading_dot: .5 + q_leading_dot: ".5" + big_int: 123456789012345678901234567890 + q_big_int: "123456789012345678901234567890" + inf: .inf + q_inf: ".inf" + nan: .NaN + q_nan: ".NaN" + bool_title: True + q_bool_title: "True" + date: 2024-01-01 + q_date: "2024-01-01" + parameters: + - name: leading_zero + default: 01 + - name: version + default: 3.10 + git_source: + git_url: https://github.com/databricks/cli + git_provider: gitHub + # 7-char short SHA with a single "e" and digits only around it + git_commit: 1234e56 + tasks: + - task_key: cluster_policy + new_cluster: + # Cluster policy ids are 16 hex characters. + policy_id: 0012345678901234 + spark_version: 13.30 + num_workers: 1 + spark_conf: + spark.sql.shuffle.partitions: 0200 + spark_env_vars: + PYTHON_VERSION: 3.10 + custom_tags: + cost_center: 0042 + notebook_task: + notebook_path: /Workspace/notebook + base_parameters: + leading_zero: 0042 + q_leading_zero: "0042" + - task_key: cluster_policy_exponent + new_cluster: + policy_id: 12345678901E0042 + spark_version: 13.3.x-scala2.12 + num_workers: 1 + notebook_task: + notebook_path: /Workspace/notebook + - task_key: existing_cluster + existing_cluster_id: 0123 + sql_task: + # SQL warehouse ids are 16 hex characters. + warehouse_id: 0123456789012345 + query: + query_id: 1e5 + + pipelines: + pipeline: + name: pipeline + catalog: 0001 + schema: 2024 + configuration: + leading_zero: 0042 + version: 1.10 + clusters: + - label: default + policy_id: 0012345678901234 + + dashboards: + dashboard: + display_name: dashboard + warehouse_id: 0123456789012345 + file_path: dashboard.lvdash.json diff --git a/acceptance/bundle/validate/yaml_numeric_strings/out.test.toml b/acceptance/bundle/validate/yaml_numeric_strings/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/validate/yaml_numeric_strings/output.txt b/acceptance/bundle/validate/yaml_numeric_strings/output.txt new file mode 100644 index 00000000000..60ca5c73bf5 --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/output.txt @@ -0,0 +1,101 @@ + +>>> [CLI] bundle validate -o json +{ + "job_name": "7", + "job_tags": { + "big_int": "123456789012345680000000000000", + "bool_title": "true", + "date": "2024-01-01", + "exponent": "1000", + "float_int": "1", + "hex": "31", + "inf": "+Inf", + "leading_dot": "0.5", + "leading_zero": "42", + "nan": "NaN", + "octal": "15", + "plus_sign": "12", + "q_big_int": "123456789012345678901234567890", + "q_bool_title": "True", + "q_date": "2024-01-01", + "q_exponent": "1e3", + "q_float_int": "1.0", + "q_hex": "0x1F", + "q_inf": ".inf", + "q_leading_dot": ".5", + "q_leading_zero": "0042", + "q_nan": ".NaN", + "q_octal": "0o17", + "q_plus_sign": "+12", + "q_trailing_zero": "1.10", + "q_underscore": "1_000", + "trailing_zero": "1.1", + "underscore": "1000" + }, + "job_parameters": [ + { + "default": "1", + "name": "leading_zero" + }, + { + "default": "3.1", + "name": "version" + } + ], + "git_commit": "123400000000000000000000000000000000000000000000000000000000", + "tasks": [ + { + "new_cluster": { + "custom_tags": { + "cost_center": "42" + }, + "policy_id": "12345678901234", + "spark_conf": { + "spark.sql.shuffle.partitions": "200" + }, + "spark_env_vars": { + "PYTHON_VERSION": "3.1" + }, + "spark_version": "13.3" + }, + "notebook_task": { + "base_parameters": { + "leading_zero": "42", + "q_leading_zero": "0042" + }, + "source": "GIT" + }, + "task_key": "cluster_policy" + }, + { + "new_cluster": { + "policy_id": "12345678901000000000000000000000000000000000000000000", + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "source": "GIT" + }, + "task_key": "cluster_policy_exponent" + }, + { + "existing_cluster_id": "123", + "sql_task": { + "query": { + "query_id": "100000" + }, + "warehouse_id": "123456789012345" + }, + "task_key": "existing_cluster" + } + ], + "pipeline": { + "catalog": "1", + "schema": "2024", + "configuration": { + "leading_zero": "42", + "version": "1.1" + }, + "policy_id": "12345678901234" + }, + "dashboard_warehouse_id": "123456789012345" +} diff --git a/acceptance/bundle/validate/yaml_numeric_strings/script b/acceptance/bundle/validate/yaml_numeric_strings/script new file mode 100644 index 00000000000..613a31a4cc2 --- /dev/null +++ b/acceptance/bundle/validate/yaml_numeric_strings/script @@ -0,0 +1,9 @@ +trace $CLI bundle validate -o json | jq '.resources | { + job_name: .jobs.job.name, + job_tags: .jobs.job.tags, + job_parameters: .jobs.job.parameters, + git_commit: .jobs.job.git_source.git_commit, + tasks: [.jobs.job.tasks[] | del(.notebook_task.notebook_path, .new_cluster.num_workers)], + pipeline: .pipelines.pipeline | {catalog, schema, configuration, policy_id: .clusters[0].policy_id}, + dashboard_warehouse_id: .dashboards.dashboard.warehouse_id +}'