diff --git a/.nextchanges/bundles/yaml-keep-source-text.md b/.nextchanges/bundles/yaml-keep-source-text.md new file mode 100644 index 00000000000..675f619ae4f --- /dev/null +++ b/.nextchanges/bundles/yaml-keep-source-text.md @@ -0,0 +1 @@ +* Keep unquoted numbers such as `policy_id: 0012345678901234` or `version: 3.10` as written when used in string fields or `${var.x}` references. Previously they were reformatted (`12345678901234`, `3.1`); values like `1.0` or `1_000` in string fields may now deploy differently. ([#7814](https://github.com/databricks/cli/pull/7814)) diff --git a/acceptance/bundle/validate/variables/numeric_defaults/output.txt b/acceptance/bundle/validate/variables/numeric_defaults/output.txt index a203b254b1b..8f235b5ef83 100644 --- a/acceptance/bundle/validate/variables/numeric_defaults/output.txt +++ b/acceptance/bundle/validate/variables/numeric_defaults/output.txt @@ -13,18 +13,18 @@ "version": 3.1 }, "max_concurrent_runs": 10, - "policy_id": "12345678901234", + "policy_id": "0012345678901234", "base_parameters": { - "exponent": "12345678901000000000000000000000000000000000000000000", + "exponent": "12345678901E0042", "from_env": "0012345678901234", "from_flag": "0012345678901234", - "interpolated": "id-1.2345678901234e+13", - "interpolated_int": "id-42", - "interpolated_version": "v3.1", - "leading_zero": "12345678901234", - "leading_zero_int": "42", + "interpolated": "id-0012345678901234", + "interpolated_int": "id-0042", + "interpolated_version": "v3.10", + "leading_zero": "0012345678901234", + "leading_zero_int": "0042", "quoted": "0012345678901234", - "target_override": "12345678901234", - "version": "3.1" + "target_override": "0012345678901234", + "version": "3.10" } } diff --git a/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml b/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml index 986b3676ece..94628c5a4e5 100644 --- a/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml +++ b/acceptance/bundle/validate/yaml_numeric_strings/databricks.yml @@ -1,7 +1,6 @@ # Unquoted scalars that YAML resolves to a number, bool or timestamp, used in # string-typed fields. Each value is written twice: unquoted and quoted ("q_"). -# The quoted value is kept verbatim; the unquoted one is converted from the -# resolved number and may not match the source text. +# Both must keep the source text verbatim. bundle: name: yaml-numeric-strings diff --git a/acceptance/bundle/validate/yaml_numeric_strings/output.txt b/acceptance/bundle/validate/yaml_numeric_strings/output.txt index 60ca5c73bf5..9a05dd5d0d8 100644 --- a/acceptance/bundle/validate/yaml_numeric_strings/output.txt +++ b/acceptance/bundle/validate/yaml_numeric_strings/output.txt @@ -1,20 +1,20 @@ >>> [CLI] bundle validate -o json { - "job_name": "7", + "job_name": "007", "job_tags": { - "big_int": "123456789012345680000000000000", + "big_int": "123456789012345678901234567890", "bool_title": "true", "date": "2024-01-01", - "exponent": "1000", - "float_int": "1", - "hex": "31", - "inf": "+Inf", - "leading_dot": "0.5", - "leading_zero": "42", - "nan": "NaN", - "octal": "15", - "plus_sign": "12", + "exponent": "1e3", + "float_int": "1.0", + "hex": "0x1F", + "inf": ".inf", + "leading_dot": ".5", + "leading_zero": "0042", + "nan": ".NaN", + "octal": "0o17", + "plus_sign": "+12", "q_big_int": "123456789012345678901234567890", "q_bool_title": "True", "q_date": "2024-01-01", @@ -29,38 +29,38 @@ "q_plus_sign": "+12", "q_trailing_zero": "1.10", "q_underscore": "1_000", - "trailing_zero": "1.1", - "underscore": "1000" + "trailing_zero": "1.10", + "underscore": "1_000" }, "job_parameters": [ { - "default": "1", + "default": "01", "name": "leading_zero" }, { - "default": "3.1", + "default": "3.10", "name": "version" } ], - "git_commit": "123400000000000000000000000000000000000000000000000000000000", + "git_commit": "1234e56", "tasks": [ { "new_cluster": { "custom_tags": { - "cost_center": "42" + "cost_center": "0042" }, - "policy_id": "12345678901234", + "policy_id": "0012345678901234", "spark_conf": { - "spark.sql.shuffle.partitions": "200" + "spark.sql.shuffle.partitions": "0200" }, "spark_env_vars": { - "PYTHON_VERSION": "3.1" + "PYTHON_VERSION": "3.10" }, - "spark_version": "13.3" + "spark_version": "13.30" }, "notebook_task": { "base_parameters": { - "leading_zero": "42", + "leading_zero": "0042", "q_leading_zero": "0042" }, "source": "GIT" @@ -69,7 +69,7 @@ }, { "new_cluster": { - "policy_id": "12345678901000000000000000000000000000000000000000000", + "policy_id": "12345678901E0042", "spark_version": "13.3.x-scala2.12" }, "notebook_task": { @@ -78,24 +78,24 @@ "task_key": "cluster_policy_exponent" }, { - "existing_cluster_id": "123", + "existing_cluster_id": "0123", "sql_task": { "query": { - "query_id": "100000" + "query_id": "1e5" }, - "warehouse_id": "123456789012345" + "warehouse_id": "0123456789012345" }, "task_key": "existing_cluster" } ], "pipeline": { - "catalog": "1", + "catalog": "0001", "schema": "2024", "configuration": { - "leading_zero": "42", - "version": "1.1" + "leading_zero": "0042", + "version": "1.10" }, - "policy_id": "12345678901234" + "policy_id": "0012345678901234" }, - "dashboard_warehouse_id": "123456789012345" + "dashboard_warehouse_id": "0123456789012345" } diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index e41ed4a0a87..ebc4943fb73 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -333,6 +333,10 @@ func (n normalizeOptions) normalizeSlice(typ reflect.Type, src dyn.Value, seen [ func (n normalizeOptions) normalizeString(typ reflect.Type, src dyn.Value, path dyn.Path) (dyn.Value, diag.Diagnostics) { var diags diag.Diagnostics + if text, ok := src.SourceText(); ok { + return dyn.NewValue(text, src.Locations()), nil + } + switch src.Kind() { case dyn.KindString: return dyn.NewValue(src.MustString(), src.Locations()), nil diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index ac95aaafac6..5d0db37c0a4 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -572,6 +572,24 @@ func TestNormalizeStringFromFloat(t *testing.T) { assert.Equal(t, dyn.NewValue("1.2", vin.Locations()), vout) } +func TestNormalizeStringFromSourceText(t *testing.T) { + var typ string + vin := dyn.NewValue(1.1, []diag.Location{{File: "file", Line: 1, Column: 1}}).WithSourceText("1.10") + vout, err := Normalize(&typ, vin) + assert.Empty(t, err) + assert.Equal(t, dyn.NewValue("1.10", vin.Locations()), vout) +} + +func TestNormalizeIntIgnoresSourceText(t *testing.T) { + var typ int + vin := dyn.V(42).WithSourceText("0042") + vout, err := Normalize(&typ, vin) + assert.Empty(t, err) + assert.Equal(t, int64(42), vout.MustInt()) + _, ok := vout.SourceText() + assert.False(t, ok) +} + func TestNormalizeStringFromTime(t *testing.T) { var typ string vin := dyn.NewValue(dyn.MustTime("2024-08-29"), []diag.Location{{File: "file", Line: 1, Column: 1}}) diff --git a/libs/dyn/convert/to_typed.go b/libs/dyn/convert/to_typed.go index 87b0fe7d8ca..aff666b7526 100644 --- a/libs/dyn/convert/to_typed.go +++ b/libs/dyn/convert/to_typed.go @@ -223,6 +223,11 @@ func toTypedSlice(dst reflect.Value, src dyn.Value) error { } func toTypedString(dst reflect.Value, src dyn.Value) error { + if text, ok := src.SourceText(); ok { + dst.SetString(text) + return nil + } + switch src.Kind() { case dyn.KindString: dst.SetString(src.MustString()) diff --git a/libs/dyn/convert/to_typed_test.go b/libs/dyn/convert/to_typed_test.go index a838dfab33f..c4e09eafe6e 100644 --- a/libs/dyn/convert/to_typed_test.go +++ b/libs/dyn/convert/to_typed_test.go @@ -389,6 +389,13 @@ func TestToTypedStringFromInt(t *testing.T) { assert.Equal(t, "123", out) } +func TestToTypedStringFromSourceText(t *testing.T) { + var out string + err := ToTyped(&out, dyn.V(1.1).WithSourceText("1.10")) + require.NoError(t, err) + assert.Equal(t, "1.10", out) +} + func TestToTypedStringFromFloat(t *testing.T) { var out string err := ToTyped(&out, dyn.V(1.2)) diff --git a/libs/dyn/dynvar/resolve.go b/libs/dyn/dynvar/resolve.go index 99981a2e51b..eeb18f28d0c 100644 --- a/libs/dyn/dynvar/resolve.go +++ b/libs/dyn/dynvar/resolve.go @@ -164,7 +164,7 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { // of where it is used. This also means that relative path resolution is done // relative to where a variable is used, not where it is defined. // - return dyn.NewValue(resolved[0].Value(), ref.Value.Locations()), nil + return resolved[0].WithLocations(ref.Value.Locations()), nil } // Not pure; perform string interpolation. @@ -184,6 +184,9 @@ func (r *resolver) resolveRef(ref Ref, seen []string) (dyn.Value, error) { // Try to turn the resolved value into a string. s, ok := resolved[j].AsString() + if !ok { + s, ok = resolved[j].SourceText() + } if !ok { // Only allow primitive types to be converted to string. switch resolved[j].Kind() { diff --git a/libs/dyn/dynvar/resolve_test.go b/libs/dyn/dynvar/resolve_test.go index c4b28e88d3f..fc48d41ead5 100644 --- a/libs/dyn/dynvar/resolve_test.go +++ b/libs/dyn/dynvar/resolve_test.go @@ -30,6 +30,22 @@ func TestResolve(t *testing.T) { assert.Equal(t, "a", getByPath(t, out, "c").MustString()) } +func TestResolveKeepsSourceText(t *testing.T) { + in := dyn.V(map[string]dyn.Value{ + "a": dyn.V(12345678901234.0).WithSourceText("0012345678901234"), + "b": dyn.V("${a}"), + "c": dyn.V("id-${a}"), + }) + + out, err := dynvar.Resolve(in, dynvar.DefaultLookup(in)) + require.NoError(t, err) + + text, ok := getByPath(t, out, "b").SourceText() + assert.True(t, ok) + assert.Equal(t, "0012345678901234", text) + assert.Equal(t, "id-0012345678901234", getByPath(t, out, "c").MustString()) +} + func TestResolveNotFound(t *testing.T) { in := dyn.V(map[string]dyn.Value{ "b": dyn.V("${a}"), diff --git a/libs/dyn/value.go b/libs/dyn/value.go index 8cbd9b801ea..acd85791a5b 100644 --- a/libs/dyn/value.go +++ b/libs/dyn/value.go @@ -21,6 +21,11 @@ type Value struct { // Whether or not this value is an anchor. // If this node doesn't map to a type, we don't need to warn about it. anchor bool + + // Source text of a scalar that the YAML loader resolved to a number, + // e.g. "0042" for the int 42. A string field uses it verbatim instead of + // formatting the resolved value, which would drop leading zeros and similar. + src string } // InvalidValue is equal to the zero-value of Value. @@ -64,6 +69,8 @@ func (v Value) WithLocations(loc []diag.Location) Value { // create a copy of the locations, so that mutations to the original slice // don't affect new value. l: slices.Clone(loc), + + src: v.src, } } @@ -72,9 +79,25 @@ func (v Value) AppendLocationsFromValue(w Value) Value { v: v.v, k: v.k, l: append(v.l, w.l...), + + src: v.src, } } +// WithSourceText returns a copy of the value that remembers the text it was parsed from. +func (v Value) WithSourceText(text string) Value { + v.src = text + return v +} + +// SourceText returns the text a number was parsed from, if it was recorded. +func (v Value) SourceText() (string, bool) { + if v.k != KindInt && v.k != KindFloat { + return "", false + } + return v.src, v.src != "" +} + func (v Value) Kind() Kind { return v.k } @@ -187,6 +210,8 @@ func (v Value) MarkAnchor() Value { l: v.l, anchor: true, + + src: v.src, } } diff --git a/libs/dyn/value_test.go b/libs/dyn/value_test.go index 667e2cbdd59..7cd80447eb7 100644 --- a/libs/dyn/value_test.go +++ b/libs/dyn/value_test.go @@ -90,3 +90,23 @@ func TestIsZero(t *testing.T) { assert.True(t, dyn.V([]dyn.Value{}).IsZero(), "Sequence") assert.False(t, dyn.V([]dyn.Value{dyn.V(0)}).IsZero(), "Sequence") } + +func TestValueSourceText(t *testing.T) { + v := dyn.V(42).WithSourceText("0042") + for _, w := range []dyn.Value{ + v, + v.WithLocations([]diag.Location{{File: "file"}}), + v.AppendLocationsFromValue(dyn.V(1)), + v.MarkAnchor(), + } { + text, ok := w.SourceText() + assert.True(t, ok) + assert.Equal(t, "0042", text) + } + + // Only numbers report source text. + _, ok := dyn.V("0042").WithSourceText("0042").SourceText() + assert.False(t, ok) + _, ok = dyn.V(true).WithSourceText("True").SourceText() + assert.False(t, ok) +} diff --git a/libs/dyn/yamlloader/loader.go b/libs/dyn/yamlloader/loader.go index 708ae5dbaac..62c6409c69d 100644 --- a/libs/dyn/yamlloader/loader.go +++ b/libs/dyn/yamlloader/loader.go @@ -209,6 +209,20 @@ func newIntValue(i64 int64, loc diag.Location) dyn.Value { } func (d *loader) loadScalar(node *yaml.Node, loc diag.Location) (dyn.Value, error) { + v, err := d.resolveScalar(node, loc) + if err != nil { + return v, err + } + switch v.Kind() { + case dyn.KindInt, dyn.KindFloat: + // Keep the source text for string fields, e.g. a policy id "0012345678901234". + return v.WithSourceText(node.Value), nil + default: + return v, nil + } +} + +func (d *loader) resolveScalar(node *yaml.Node, loc diag.Location) (dyn.Value, error) { st := node.ShortTag() switch st { case "!!str": diff --git a/libs/dyn/yamlloader/yaml_test.go b/libs/dyn/yamlloader/yaml_test.go index e21065e92c0..ce6924bf881 100644 --- a/libs/dyn/yamlloader/yaml_test.go +++ b/libs/dyn/yamlloader/yaml_test.go @@ -33,3 +33,20 @@ func TestYAMLEmpty(t *testing.T) { self := loadYAML(t, "testdata/empty.yml") assert.Equal(t, dyn.NilValue, self) } + +func TestYAMLScalarSourceText(t *testing.T) { + input := "int: 0042\nneg: -007\nfloat: 1.10\nalias: &a 0042\nref: *a\nbool: True\nquoted: \"0042\"\nstr: abc\nnull: ~\ndate: 2024-01-01\n" + v, err := yamlloader.LoadYAML("file.yml", bytes.NewBufferString(input)) + require.NoError(t, err) + + for key, want := range map[string]string{"int": "0042", "neg": "-007", "float": "1.10", "alias": "0042", "ref": "0042"} { + text, ok := v.Get(key).SourceText() + assert.True(t, ok, key) + assert.Equal(t, want, text, key) + } + + for _, key := range []string{"bool", "quoted", "str", "null", "date"} { + _, ok := v.Get(key).SourceText() + assert.False(t, ok, key) + } +}