Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/yaml-keep-source-text.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Keep unquoted numbers such as `policy_id: 0012345678901234` or `version: 3.10` as written when used in string fields or `${var.x}` references. Previously they were reformatted (`12345678901234`, `3.1`); values like `1.0` or `1_000` in string fields may now deploy differently. ([#7814](https://github.com/databricks/cli/pull/7814))
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,18 @@
"version": 3.1
},
"max_concurrent_runs": 10,
"policy_id": "12345678901234",
"policy_id": "0012345678901234",
"base_parameters": {
"exponent": "12345678901000000000000000000000000000000000000000000",
"exponent": "12345678901E0042",
"from_env": "0012345678901234",
"from_flag": "0012345678901234",
"interpolated": "id-1.2345678901234e+13",
"interpolated_int": "id-42",
"interpolated_version": "v3.1",
"leading_zero": "12345678901234",
"leading_zero_int": "42",
"interpolated": "id-0012345678901234",
"interpolated_int": "id-0042",
"interpolated_version": "v3.10",
"leading_zero": "0012345678901234",
"leading_zero_int": "0042",
"quoted": "0012345678901234",
"target_override": "12345678901234",
"version": "3.1"
"target_override": "0012345678901234",
"version": "3.10"
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# Unquoted scalars that YAML resolves to a number, bool or timestamp, used in
# string-typed fields. Each value is written twice: unquoted and quoted ("q_").
# The quoted value is kept verbatim; the unquoted one is converted from the
# resolved number and may not match the source text.
# Both must keep the source text verbatim.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI Generated] nit: bool_title: True on line 36 still comes out as "true" (output.txt:7), so "Both must keep the source text verbatim" does not hold for bools. Can the comment say only numbers (and timestamps) are kept verbatim?

bundle:
name: yaml-numeric-strings

Expand Down
62 changes: 31 additions & 31 deletions acceptance/bundle/validate/yaml_numeric_strings/output.txt
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@

>>> [CLI] bundle validate -o json
{
"job_name": "7",
"job_name": "007",
"job_tags": {
"big_int": "123456789012345680000000000000",
"big_int": "123456789012345678901234567890",
"bool_title": "true",
"date": "2024-01-01",
"exponent": "1000",
"float_int": "1",
"hex": "31",
"inf": "+Inf",
"leading_dot": "0.5",
"leading_zero": "42",
"nan": "NaN",
"octal": "15",
"plus_sign": "12",
"exponent": "1e3",
"float_int": "1.0",
"hex": "0x1F",
"inf": ".inf",
"leading_dot": ".5",
"leading_zero": "0042",
"nan": ".NaN",
"octal": "0o17",
"plus_sign": "+12",
"q_big_int": "123456789012345678901234567890",
"q_bool_title": "True",
"q_date": "2024-01-01",
Expand All @@ -29,38 +29,38 @@
"q_plus_sign": "+12",
"q_trailing_zero": "1.10",
"q_underscore": "1_000",
"trailing_zero": "1.1",
"underscore": "1000"
"trailing_zero": "1.10",
"underscore": "1_000"
},
"job_parameters": [
{
"default": "1",
"default": "01",
"name": "leading_zero"
},
{
"default": "3.1",
"default": "3.10",
"name": "version"
}
],
"git_commit": "123400000000000000000000000000000000000000000000000000000000",
"git_commit": "1234e56",
"tasks": [
{
"new_cluster": {
"custom_tags": {
"cost_center": "42"
"cost_center": "0042"
},
"policy_id": "12345678901234",
"policy_id": "0012345678901234",
"spark_conf": {
"spark.sql.shuffle.partitions": "200"
"spark.sql.shuffle.partitions": "0200"
},
"spark_env_vars": {
"PYTHON_VERSION": "3.1"
"PYTHON_VERSION": "3.10"
},
"spark_version": "13.3"
"spark_version": "13.30"
},
"notebook_task": {
"base_parameters": {
"leading_zero": "42",
"leading_zero": "0042",
"q_leading_zero": "0042"
},
"source": "GIT"
Expand All @@ -69,7 +69,7 @@
},
{
"new_cluster": {
"policy_id": "12345678901000000000000000000000000000000000000000000",
"policy_id": "12345678901E0042",
"spark_version": "13.3.x-scala2.12"
},
"notebook_task": {
Expand All @@ -78,24 +78,24 @@
"task_key": "cluster_policy_exponent"
},
{
"existing_cluster_id": "123",
"existing_cluster_id": "0123",
"sql_task": {
"query": {
"query_id": "100000"
"query_id": "1e5"
},
"warehouse_id": "123456789012345"
"warehouse_id": "0123456789012345"
},
"task_key": "existing_cluster"
}
],
"pipeline": {
"catalog": "1",
"catalog": "0001",
"schema": "2024",
"configuration": {
"leading_zero": "42",
"version": "1.1"
"leading_zero": "0042",
"version": "1.10"
},
"policy_id": "12345678901234"
"policy_id": "0012345678901234"
},
"dashboard_warehouse_id": "123456789012345"
"dashboard_warehouse_id": "0123456789012345"
}
4 changes: 4 additions & 0 deletions libs/dyn/convert/normalize.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,10 @@ func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen [
func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) {
var diags diag.Diagnostics

if text, ok := src.SourceText(); ok {
return dyn.NewValue(text, src.Locations()), nil
}

switch src.Kind() {
case dyn.KindString:
return dyn.NewValue(src.MustString(), src.Locations()), nil
Expand Down
18 changes: 18 additions & 0 deletions libs/dyn/convert/normalize_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,24 @@ func TestNormalizeStringFromFloat(t *testing.T) {
assert.Equal(t, dyn.NewValue("1.2", vin.Locations()), vout)
}

func TestNormalizeStringFromSourceText(t *testing.T) {
var typ string
vin := dyn.NewValue(1.1, []diag.Location{{File: "file", Line: 1, Column: 1}}).WithSourceText("1.10")
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, dyn.NewValue("1.10", vin.Locations()), vout)
}

func TestNormalizeIntIgnoresSourceText(t *testing.T) {
var typ int
vin := dyn.V(42).WithSourceText("0042")
vout, err := Normalize(&typ, vin)
assert.Empty(t, err)
assert.Equal(t, int64(42), vout.MustInt())
_, ok := vout.SourceText()
assert.False(t, ok)
}

func TestNormalizeStringFromTime(t *testing.T) {
var typ string
vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []diag.Location{{File: "file", Line: 1, Column: 1}})
Expand Down
5 changes: 5 additions & 0 deletions libs/dyn/convert/to_typed.go
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,11 @@ func toTypedSlice(dst reflect.Value, src dyn.Value) error {
}

func toTypedString(dst reflect.Value, src dyn.Value) error {
if text, ok := src.SourceText(); ok {
dst.SetString(text)
return nil
}

switch src.Kind() {
case dyn.KindString:
dst.SetString(src.MustString())
Expand Down
7 changes: 7 additions & 0 deletions libs/dyn/convert/to_typed_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,13 @@ func TestToTypedStringFromInt(t *testing.T) {
assert.Equal(t, "123", out)
}

func TestToTypedStringFromSourceText(t *testing.T) {
var out string
err := ToTyped(&out, dyn.V(1.1).WithSourceText("1.10"))
require.NoError(t, err)
assert.Equal(t, "1.10", out)
}

func TestToTypedStringFromFloat(t *testing.T) {
var out string
err := ToTyped(&out, dyn.V(1.2))
Expand Down
5 changes: 4 additions & 1 deletion libs/dyn/dynvar/resolve.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) {
// of where it is used. This also means that relative path resolution is done
// relative to where a variable is used, not where it is defined.
//
return dyn.NewValue(resolved[0].Value(), ref.Value.Locations()), nil
return resolved[0].WithLocations(ref.Value.Locations()), nil
}

// Not pure; perform string interpolation.
Expand All @@ -184,6 +184,9 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) {

// Try to turn the resolved value into a string.
s, ok := resolved[j].AsString()
if !ok {
s, ok = resolved[j].SourceText()
}
if !ok {
// Only allow primitive types to be converted to string.
switch resolved[j].Kind() {
Expand Down
16 changes: 16 additions & 0 deletions libs/dyn/dynvar/resolve_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,22 @@ func TestResolve(t *testing.T) {
assert.Equal(t, "a", getByPath(t, out, "c").MustString())
}

func TestResolveKeepsSourceText(t *testing.T) {
in := dyn.V(map[string]dyn.Value{
"a": dyn.V(12345678901234.0).WithSourceText("0012345678901234"),
"b": dyn.V("${a}"),
"c": dyn.V("id-${a}"),
})

out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in))
require.NoError(t, err)

text, ok := getByPath(t, out, "b").SourceText()
assert.True(t, ok)
assert.Equal(t, "0012345678901234", text)
assert.Equal(t, "id-0012345678901234", getByPath(t, out, "c").MustString())
}

func TestResolveNotFound(t *testing.T) {
in := dyn.V(map[string]dyn.Value{
"b": dyn.V("${a}"),
Expand Down
25 changes: 25 additions & 0 deletions libs/dyn/value.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ type Value struct {
// Whether or not this value is an anchor.
// If this node doesn't map to a type, we don't need to warn about it.
anchor bool

// Source text of a scalar that the YAML loader resolved to a number,
// e.g. "0042" for the int 42. A string field uses it verbatim instead of
// formatting the resolved value, which would drop leading zeros and similar.
src string
}

// InvalidValue is equal to the zero-value of Value.
Expand Down Expand Up @@ -64,6 +69,8 @@ func (v Value) WithLocations(loc []diag.Location) Value {
// create a copy of the locations, so that mutations to the original slice
// don't affect new value.
l: slices.Clone(loc),

src: v.src,
}
}

Expand All @@ -72,9 +79,25 @@ func (v Value) AppendLocationsFromValue(w Value) Value {
v: v.v,
k: v.k,
l: append(v.l, w.l...),

src: v.src,
}
}

// WithSourceText returns a copy of the value that remembers the text it was parsed from.
func (v Value) WithSourceText(text string) Value {
v.src = text
return v
}

// SourceText returns the text a number was parsed from, if it was recorded.
func (v Value) SourceText() (string, bool) {
if v.k != KindInt && v.k != KindFloat {
return "", false
}
return v.src, v.src != ""
}

func (v Value) Kind() Kind {
return v.k
}
Expand Down Expand Up @@ -187,6 +210,8 @@ func (v Value) MarkAnchor() Value {
l: v.l,

anchor: true,

src: v.src,
}
}

Expand Down
20 changes: 20 additions & 0 deletions libs/dyn/value_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,3 +90,23 @@ func TestIsZero(t *testing.T) {
assert.True(t, dyn.V([]dyn.Value{}).IsZero(), "Sequence")
assert.False(t, dyn.V([]dyn.Value{dyn.V(0)}).IsZero(), "Sequence")
}

func TestValueSourceText(t *testing.T) {
v := dyn.V(42).WithSourceText("0042")
for _, w := range []dyn.Value{
v,
v.WithLocations([]diag.Location{{File: "file"}}),
v.AppendLocationsFromValue(dyn.V(1)),
v.MarkAnchor(),
} {
text, ok := w.SourceText()
assert.True(t, ok)
assert.Equal(t, "0042", text)
}

// Only numbers report source text.
_, ok := dyn.V("0042").WithSourceText("0042").SourceText()
assert.False(t, ok)
_, ok = dyn.V(true).WithSourceText("True").SourceText()
assert.False(t, ok)
}
14 changes: 14 additions & 0 deletions libs/dyn/yamlloader/loader.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,20 @@ func newIntValue(i64 int64, loc diag.Location) dyn.Value {
}

func (d *loader) loadScalar(node *yaml.Node, loc diag.Location) (dyn.Value, error) {
v, err := d.resolveScalar(node, loc)
if err != nil {
return v, err
}
switch v.Kind() {
case dyn.KindInt, dyn.KindFloat:
// Keep the source text for string fields, e.g. a policy id "0012345678901234".
return v.WithSourceText(node.Value), nil
default:
return v, nil
}
}

func (d *loader) resolveScalar(node *yaml.Node, loc diag.Location) (dyn.Value, error) {
st := node.ShortTag()
switch st {
case "!!str":
Expand Down
17 changes: 17 additions & 0 deletions libs/dyn/yamlloader/yaml_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,20 @@ func TestYAMLEmpty(t *testing.T) {
self := loadYAML(t, "testdata/empty.yml")
assert.Equal(t, dyn.NilValue, self)
}

func TestYAMLScalarSourceText(t *testing.T) {
input := "int: 0042\nneg: -007\nfloat: 1.10\nalias: &a 0042\nref: *a\nbool: True\nquoted: \"0042\"\nstr: abc\nnull: ~\ndate: 2024-01-01\n"
v, err := yamlloader.LoadYAML("file.yml", bytes.NewBufferString(input))
require.NoError(t, err)

for key, want := range map[string]string{"int": "0042", "neg": "-007", "float": "1.10", "alias": "0042", "ref": "0042"} {
text, ok := v.Get(key).SourceText()
assert.True(t, ok, key)
assert.Equal(t, want, text, key)
}

for _, key := range []string{"bool", "quoted", "str", "null", "date"} {
_, ok := v.Get(key).SourceText()
assert.False(t, ok, key)
}
}
Loading