Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Variable defaults that YAML resolves to a number, referenced from string fields.
bundle:
name: numeric-defaults

variables:
# YAML resolves this to a float: with a leading "0" it is not a valid octal int (has 8, 9).

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI Generated] nit: The float comes from yaml.v3, not YAML: the 1.2 core schema reads 0012345678901234 as an int and PyYAML keeps it a string. Can this say "yaml.v3 resolves"?

leading_zero:
default: 0012345678901234
# YAML resolves this to an int (valid octal digits only); the loader parses it as decimal 42.
leading_zero_int:
default: 0042
exponent:
default: 12345678901E0042
version:
default: 3.10
quoted:
default: "0012345678901234"
# Used in an int field, which must get the parsed number.
concurrency:
default: 010
# Overridden in the target below, see targets.dev.variables.
target_override:
default: "0012345678901234"
# Overridden with --var and BUNDLE_VAR_ in the script.
from_flag:
default: placeholder
from_env:
default: placeholder

resources:
jobs:
job:
name: job
max_concurrent_runs: ${var.concurrency}
tasks:
- task_key: t
new_cluster:
policy_id: ${var.leading_zero}
spark_version: 13.3.x-scala2.12
num_workers: 1
notebook_task:
notebook_path: /Workspace/notebook
base_parameters:
leading_zero: ${var.leading_zero}
exponent: ${var.exponent}
version: ${var.version}
quoted: ${var.quoted}
target_override: ${var.target_override}
from_flag: ${var.from_flag}
from_env: ${var.from_env}
leading_zero_int: ${var.leading_zero_int}
interpolated: id-${var.leading_zero}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI Generated] id-${var.leading_zero} renders as id-1.2345678901234e+13, while the pure reference gives 12345678901234. If that's a bug, can a test.toml Badness line record it, like target_timestamp_shorthand does?

interpolated_int: id-${var.leading_zero_int}
interpolated_version: v${var.version}

targets:
dev:
default: true
variables:
target_override: 0012345678901234
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@

>>> [CLI] bundle validate -o json --var from_flag=0012345678901234
{
"variables": {
"concurrency": 10,
"exponent": 1.2345678901E+52,
"from_env": "0012345678901234",
"from_flag": "0012345678901234",
"leading_zero": 12345678901234,
"leading_zero_int": 42,
"quoted": "0012345678901234",
"target_override": 12345678901234,
"version": 3.1
},
"max_concurrent_runs": 10,
"policy_id": "12345678901234",
"base_parameters": {
"exponent": "12345678901000000000000000000000000000000000000000000",
"from_env": "0012345678901234",
"from_flag": "0012345678901234",
"interpolated": "id-1.2345678901234e+13",
"interpolated_int": "id-42",
"interpolated_version": "v3.1",
"leading_zero": "12345678901234",
"leading_zero_int": "42",
"quoted": "0012345678901234",
"target_override": "12345678901234",
"version": "3.1"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
export BUNDLE_VAR_from_env=0012345678901234
trace $CLI bundle validate -o json --var from_flag=0012345678901234 | jq '{
variables: .variables | map_values(.value),
max_concurrent_runs: .resources.jobs.job.max_concurrent_runs,
policy_id: .resources.jobs.job.tasks[0].new_cluster.policy_id,
base_parameters: .resources.jobs.job.tasks[0].notebook_task.base_parameters
}'
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
100 changes: 100 additions & 0 deletions acceptance/bundle/validate/yaml_numeric_strings/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
# Unquoted scalars that YAML resolves to a number, bool or timestamp, used in
# string-typed fields. Each value is written twice: unquoted and quoted ("q_").

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI Generated] nit: Only tags and base_parameters have q_ pairs; name, parameters, git_commit, the ids and the pipeline fields are written once, unquoted. Can the header say that instead of "each value is written twice"?

# The quoted value is kept verbatim; the unquoted one is converted from the
# resolved number and may not match the source text.
bundle:
name: yaml-numeric-strings

resources:
jobs:
job:
name: 007
tags:
leading_zero: 0042
q_leading_zero: "0042"
trailing_zero: 1.10
q_trailing_zero: "1.10"
float_int: 1.0
q_float_int: "1.0"
exponent: 1e3
q_exponent: "1e3"
hex: 0x1F
q_hex: "0x1F"
octal: 0o17
q_octal: "0o17"
underscore: 1_000
q_underscore: "1_000"
plus_sign: +12
q_plus_sign: "+12"
leading_dot: .5
q_leading_dot: ".5"
big_int: 123456789012345678901234567890
q_big_int: "123456789012345678901234567890"
inf: .inf
q_inf: ".inf"
nan: .NaN
q_nan: ".NaN"
bool_title: True
q_bool_title: "True"
date: 2024-01-01
q_date: "2024-01-01"
parameters:
- name: leading_zero
default: 01
- name: version
default: 3.10
git_source:
git_url: https://github.com/databricks/cli
git_provider: gitHub
# 7-char short SHA with a single "e" and digits only around it
git_commit: 1234e56
tasks:
- task_key: cluster_policy
new_cluster:
# Cluster policy ids are 16 hex characters.
policy_id: 0012345678901234
spark_version: 13.30
num_workers: 1
spark_conf:
spark.sql.shuffle.partitions: 0200
spark_env_vars:
PYTHON_VERSION: 3.10
custom_tags:
cost_center: 0042
notebook_task:
notebook_path: /Workspace/notebook
base_parameters:
leading_zero: 0042
q_leading_zero: "0042"
- task_key: cluster_policy_exponent
new_cluster:
policy_id: 12345678901E0042
spark_version: 13.3.x-scala2.12
num_workers: 1
notebook_task:
notebook_path: /Workspace/notebook
- task_key: existing_cluster
existing_cluster_id: 0123
sql_task:
# SQL warehouse ids are 16 hex characters.
warehouse_id: 0123456789012345
query:
query_id: 1e5

pipelines:
pipeline:
name: pipeline
catalog: 0001
schema: 2024
configuration:
leading_zero: 0042
version: 1.10
clusters:
- label: default
policy_id: 0012345678901234

dashboards:
dashboard:
display_name: dashboard
warehouse_id: 0123456789012345
file_path: dashboard.lvdash.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
101 changes: 101 additions & 0 deletions acceptance/bundle/validate/yaml_numeric_strings/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@

>>> [CLI] bundle validate -o json
{
"job_name": "7",
"job_tags": {
"big_int": "123456789012345680000000000000",
"bool_title": "true",
"date": "2024-01-01",
"exponent": "1000",
"float_int": "1",
"hex": "31",
"inf": "+Inf",
"leading_dot": "0.5",
"leading_zero": "42",
"nan": "NaN",
"octal": "15",
"plus_sign": "12",
"q_big_int": "123456789012345678901234567890",
"q_bool_title": "True",
"q_date": "2024-01-01",
"q_exponent": "1e3",
"q_float_int": "1.0",
"q_hex": "0x1F",
"q_inf": ".inf",
"q_leading_dot": ".5",
"q_leading_zero": "0042",
"q_nan": ".NaN",
"q_octal": "0o17",
"q_plus_sign": "+12",
"q_trailing_zero": "1.10",
"q_underscore": "1_000",
"trailing_zero": "1.1",
"underscore": "1000"
},
"job_parameters": [
{
"default": "1",
"name": "leading_zero"
},
{
"default": "3.1",
"name": "version"
}
],
"git_commit": "123400000000000000000000000000000000000000000000000000000000",
"tasks": [
{
"new_cluster": {
"custom_tags": {
"cost_center": "42"
},
"policy_id": "12345678901234",
"spark_conf": {
"spark.sql.shuffle.partitions": "200"
},
"spark_env_vars": {
"PYTHON_VERSION": "3.1"
},
"spark_version": "13.3"
},
"notebook_task": {
"base_parameters": {
"leading_zero": "42",
"q_leading_zero": "0042"
},
"source": "GIT"
},
"task_key": "cluster_policy"
},
{
"new_cluster": {
"policy_id": "12345678901000000000000000000000000000000000000000000",
"spark_version": "13.3.x-scala2.12"
},
"notebook_task": {
"source": "GIT"
},
"task_key": "cluster_policy_exponent"
},
{
"existing_cluster_id": "123",
"sql_task": {
"query": {
"query_id": "100000"
},
"warehouse_id": "123456789012345"
},
"task_key": "existing_cluster"
}
],
"pipeline": {
"catalog": "1",
"schema": "2024",
"configuration": {
"leading_zero": "42",
"version": "1.1"
},
"policy_id": "12345678901234"
},
"dashboard_warehouse_id": "123456789012345"
}
9 changes: 9 additions & 0 deletions acceptance/bundle/validate/yaml_numeric_strings/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
trace $CLI bundle validate -o json | jq '.resources | {
job_name: .jobs.job.name,
job_tags: .jobs.job.tags,
job_parameters: .jobs.job.parameters,
git_commit: .jobs.job.git_source.git_commit,
tasks: [.jobs.job.tasks[] | del(.notebook_task.notebook_path, .new_cluster.num_workers)],
pipeline: .pipelines.pipeline | {catalog, schema, configuration, policy_id: .clusters[0].policy_id},
dashboard_warehouse_id: .dashboards.dashboard.warehouse_id
}'
Loading