From 06ec194b5918712a61a5398f9d1d3565bece0c57 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 19 Nov 2026 01:05:55 +0000 Subject: [PATCH 1/9] libs: add structcopy to copy the fields two structs share The SDK describes one resource with several structs: a response, a create request, an update request. A field they share has the same Go name, JSON name and type in each, so a Copier matches fields by name and is told about the exceptions only (Skip, ForceSend). ForceSendFields is derived rather than copied: it lists the copied fields that the source lists, plus the fields declared with ForceSend. The SDK marshaller rejects a name its struct lacks and would send the zero value of a field that was left unset, which is what utils.FilterFields is called for by hand today. Describe lists what a Copier does with each destination field, for a golden file that shows the effect of an SDK bump. Co-authored-by: Isaac --- libs/structs/structcopy/copy.go | 337 ++++++++++++++++++++++++ libs/structs/structcopy/copy_test.go | 372 +++++++++++++++++++++++++++ 2 files changed, 709 insertions(+) create mode 100644 libs/structs/structcopy/copy.go create mode 100644 libs/structs/structcopy/copy_test.go diff --git a/libs/structs/structcopy/copy.go b/libs/structs/structcopy/copy.go new file mode 100644 index 00000000000..d204baf1092 --- /dev/null +++ b/libs/structs/structcopy/copy.go @@ -0,0 +1,337 @@ +// Package structcopy copies the fields two struct types have in common. +// +// The SDK describes one resource with several structs: a response, a create request, an +// update request. A field they share has the same Go name, JSON name and type in each, so +// a [Copier] matches fields by name and is told about the exceptions only. A field the SDK +// adds to both structs is copied without a code change. +package structcopy + +import ( + "fmt" + "maps" + "reflect" + "slices" + "strings" + "sync" + + "github.com/databricks/cli/libs/structs/structtag" +) + +const forceSendFieldsName = "ForceSendFields" + +// Copier copies the fields that Src and Dst have in common. +type Copier[Src, Dst any] struct { + plan func() (*plan, error) +} + +type options struct { + skip []string + forceSend []string +} + +// Option names destination fields that a [Copier] treats differently. +type Option func(*options) + +// Skip leaves the named destination fields unset although the source has them. +func Skip(fields ...string) Option { + return func(o *options) { + o.skip = append(o.skip, fields...) + } +} + +// ForceSend lists the named destination fields in ForceSendFields on every copy, so the +// SDK sends them even when they are zero. +func ForceSend(fields ...string) Option { + return func(o *options) { + o.forceSend = append(o.forceSend, fields...) + } +} + +// New returns a Copier from Src to Dst. The types are inspected on first use, so declaring +// a Copier in a package-level variable costs nothing at startup. An invalid Copier makes +// [Copier.Copy] panic: call [Copier.Describe] from a test to find out before it runs. +func New[Src, Dst any](opts ...Option) *Copier[Src, Dst] { + return &Copier[Src, Dst]{ + plan: sync.OnceValues(func() (*plan, error) { + var o options + for _, opt := range opts { + opt(&o) + } + src, dst := reflect.TypeFor[Src](), reflect.TypeFor[Dst]() + p, err := newPlan(src, dst, o) + if err != nil { + return nil, fmt.Errorf("structcopy: %s -> %s: %w", src, dst, err) + } + return p, nil + }), + } +} + +// Copy returns a new Dst holding the fields of src that Dst has as well. The copy is +// shallow: maps, slices and pointers are shared with src. +// +// ForceSendFields is not copied but derived: it lists the copied fields that src lists, +// followed by the [ForceSend] fields. The SDK marshaller rejects a name its struct lacks +// and would send the zero value of a field that was left unset. +func (c *Copier[Src, Dst]) Copy(src *Src) *Dst { + p, err := c.plan() + if err != nil { + panic(err) + } + dst := new(Dst) + p.apply(reflect.ValueOf(src).Elem(), reflect.ValueOf(dst).Elem()) + return dst +} + +// Describe lists what the Copier does with every field of the destination, one line per +// field, or reports why the Copier is invalid. +func (c *Copier[Src, Dst]) Describe() (string, error) { + p, err := c.plan() + if err != nil { + return "", err + } + var b strings.Builder + fmt.Fprintf(&b, "%s -> %s\n", p.src, p.dst) + for _, f := range p.fields { + fmt.Fprintf(&b, " %s: %s\n", f.name, f.outcome) + } + return b.String(), nil +} + +// plan is what a Copier does, worked out once from the two types. +type plan struct { + src, dst reflect.Type + + // fields describes every destination field, sorted by name. + fields []fieldOutcome + steps []copyStep + forced []forceSendSlot +} + +type fieldOutcome struct { + name string + outcome string +} + +type copyStep struct { + src, dst []int + + // convertTo is set when the field types differ, as with the enum types the SDK + // declares once per request and response. + convertTo reflect.Type +} + +// forceSendSlot is one ForceSendFields of the destination. There can be several, since a +// struct that embeds another keeps a ForceSendFields per struct. +type forceSendSlot struct { + index []int + always []string + from []forceSendSource +} + +// forceSendSource is a ForceSendFields of the source together with the fields it governs +// that were copied into the slot. +type forceSendSource struct { + index []int + copied []string +} + +func newPlan(src, dst reflect.Type, o options) (*plan, error) { + srcFields, err := fieldsOf(src) + if err != nil { + return nil, err + } + dstFields, err := fieldsOf(dst) + if err != nil { + return nil, err + } + + for _, name := range o.skip { + if _, ok := dstFields[name]; !ok { + return nil, fmt.Errorf("cannot skip %s: the destination has no such field", name) + } + if _, ok := srcFields[name]; !ok { + return nil, fmt.Errorf("cannot skip %s: the source has no such field, so it is not copied anyway", name) + } + } + + p := &plan{src: src, dst: dst, fields: nil, steps: nil, forced: nil} + + for _, name := range o.forceSend { + d, ok := dstFields[name] + if !ok { + return nil, fmt.Errorf("cannot force-send %s: the destination has no such field", name) + } + if !isBasic(d.typ.Kind()) { + return nil, fmt.Errorf("cannot force-send %s: ForceSendFields has no effect on a %s", name, d.typ.Kind()) + } + if d.forceSend == nil { + return nil, fmt.Errorf("cannot force-send %s: the struct that declares it has no ForceSendFields", name) + } + slot := p.slot(d.forceSend) + slot.always = append(slot.always, name) + } + + for _, name := range slices.Sorted(maps.Keys(dstFields)) { + d := dstFields[name] + s, inSource := srcFields[name] + + var outcome string + switch { + case slices.Contains(o.skip, name): + outcome = "skipped" + case !inSource: + outcome = "not in source" + default: + step, err := newCopyStep(s, d) + if err != nil { + return nil, err + } + p.steps = append(p.steps, step) + outcome = "copied" + if step.convertTo != nil { + outcome += ", converted from " + s.typ.String() + } + if s.forceSend != nil && d.forceSend != nil { + p.slot(d.forceSend).copiedFrom(s.forceSend, name) + } + } + if slices.Contains(o.forceSend, name) { + outcome += ", always sent" + } + p.fields = append(p.fields, fieldOutcome{name: name, outcome: outcome}) + } + + return p, nil +} + +func newCopyStep(s, d field) (copyStep, error) { + step := copyStep{src: s.index, dst: d.index, convertTo: nil} + if s.jsonName != d.jsonName { + return step, fmt.Errorf("%s is %q in the source's JSON but %q in the destination's; skip it and set it by hand", d.name, s.jsonName, d.jsonName) + } + if s.typ == d.typ { + return step, nil + } + if s.typ.Kind() != d.typ.Kind() || !isBasic(d.typ.Kind()) { + return step, fmt.Errorf("%s is %s in the source but %s in the destination; skip it and set it by hand", d.name, s.typ, d.typ) + } + step.convertTo = d.typ + return step, nil +} + +// slot returns the forceSendSlot for the destination's ForceSendFields at index. +func (p *plan) slot(index []int) *forceSendSlot { + for i := range p.forced { + if slices.Equal(p.forced[i].index, index) { + return &p.forced[i] + } + } + p.forced = append(p.forced, forceSendSlot{index: index, always: nil, from: nil}) + return &p.forced[len(p.forced)-1] +} + +// copiedFrom records that the named field is copied into the slot from a struct whose +// ForceSendFields is at index. +func (s *forceSendSlot) copiedFrom(index []int, name string) { + for i := range s.from { + if slices.Equal(s.from[i].index, index) { + s.from[i].copied = append(s.from[i].copied, name) + return + } + } + s.from = append(s.from, forceSendSource{index: index, copied: []string{name}}) +} + +func (p *plan) apply(src, dst reflect.Value) { + for _, step := range p.steps { + v := src.FieldByIndex(step.src) + if step.convertTo != nil { + v = v.Convert(step.convertTo) + } + dst.FieldByIndex(step.dst).Set(v) + } + + for _, slot := range p.forced { + names := slices.Clone(slot.always) + for _, from := range slot.from { + listed, _ := reflect.TypeAssert[[]string](src.FieldByIndex(from.index)) + for _, name := range listed { + if slices.Contains(from.copied, name) && !slices.Contains(names, name) { + names = append(names, name) + } + } + } + if len(names) > 0 { + dst.FieldByIndex(slot.index).Set(reflect.ValueOf(names)) + } + } +} + +// field is a field that a composite literal of its struct could set by name: declared by +// the struct itself or promoted from a struct it embeds. +type field struct { + name string + index []int + typ reflect.Type + jsonName string + + // forceSend is the index of the ForceSendFields that governs the field: the one + // declared by the same struct. It is nil if that struct declares none. + forceSend []int +} + +func fieldsOf(t reflect.Type) (map[string]field, error) { + if t.Kind() != reflect.Struct { + return nil, fmt.Errorf("%s is not a struct", t) + } + + fields := map[string]field{} + for _, sf := range reflect.VisibleFields(t) { + if sf.Anonymous { + if sf.Type.Kind() == reflect.Struct { + // Its fields are promoted, so they are visited on their own. + continue + } + if sf.Type.Kind() == reflect.Pointer && sf.Type.Elem().Kind() == reflect.Struct { + return nil, fmt.Errorf("%s embeds %s; embedded pointers are not supported", t, sf.Type) + } + } + if !sf.IsExported() || sf.Name == forceSendFieldsName { + continue + } + + parentIndex := sf.Index[:len(sf.Index)-1] + parent := t + if len(parentIndex) > 0 { + parent = t.FieldByIndex(parentIndex).Type + } + var forceSend []int + if fsf, ok := parent.FieldByName(forceSendFieldsName); ok && len(fsf.Index) == 1 && fsf.Type == reflect.TypeFor[[]string]() { + forceSend = append(slices.Clone(parentIndex), fsf.Index[0]) + } + + fields[sf.Name] = field{ + name: sf.Name, + index: sf.Index, + typ: sf.Type, + jsonName: structtag.JSONTag(sf.Tag.Get("json")).Name(), + forceSend: forceSend, + } + } + return fields, nil +} + +// isBasic reports whether ForceSendFields applies to a field of this kind; see isBasicType +// in the SDK's marshal package. +func isBasic(kind reflect.Kind) bool { + switch kind { + case reflect.Bool, reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64: + return true + default: + return false + } +} diff --git a/libs/structs/structcopy/copy_test.go b/libs/structs/structcopy/copy_test.go new file mode 100644 index 00000000000..be4c7ac0116 --- /dev/null +++ b/libs/structs/structcopy/copy_test.go @@ -0,0 +1,372 @@ +package structcopy_test + +import ( + "testing" + + "github.com/databricks/cli/libs/structs/structcopy" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The fixtures mirror how the SDK shapes one resource: a response, a create request the +// state embeds, and an update request whose path parameter reuses a body field's name. + +type ( + remoteKind string + createKind string +) + +type remoteThing struct { + Comment string `json:"comment,omitempty"` + CreatedAt int64 `json:"created_at,omitempty"` + Kind remoteKind `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Size int `json:"size,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + ThingId string `json:"thing_id,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` +} + +type createThing struct { + Comment string `json:"comment,omitempty"` + Kind createKind `json:"kind,omitempty"` + Name string `json:"name,omitempty"` + Size int `json:"size,omitempty"` + SkipValidation bool `json:"skip_validation,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` +} + +type thingState struct { + createThing + + ThingId string `json:"thing_id,omitempty"` +} + +type updateThing struct { + Comment string `json:"comment,omitempty"` + // Name is the path parameter that addresses the thing, not the body field of createThing. + Name string `json:"-" url:"-"` + NewName string `json:"new_name,omitempty"` + Owner string `json:"owner,omitempty"` + Size int `json:"size,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` +} + +func TestCopySharedFields(t *testing.T) { + src := &remoteThing{ + Comment: "a comment", + CreatedAt: 1700000000, + Kind: "LARGE", + Name: "thing", + Size: 3, + Tags: map[string]string{"team": "data"}, + ThingId: "id-1", + } + + got := structcopy.New[remoteThing, createThing]().Copy(src) + + assert.Equal(t, &createThing{ + Comment: "a comment", + Kind: "LARGE", + Name: "thing", + Size: 3, + Tags: map[string]string{"team": "data"}, + }, got) +} + +func TestCopyKeepsForceSendFieldsOfCopiedFields(t *testing.T) { + src := &remoteThing{ + // CreatedAt and ThingId have no counterpart in createThing; the SDK marshaller + // rejects a ForceSendFields name its struct does not have. + ForceSendFields: []string{"Comment", "CreatedAt", "Size", "ThingId"}, + } + + got := structcopy.New[remoteThing, createThing]().Copy(src) + + assert.Equal(t, []string{"Comment", "Size"}, got.ForceSendFields) +} + +func TestCopyWithoutForcedFieldsLeavesForceSendFieldsNil(t *testing.T) { + got := structcopy.New[remoteThing, createThing]().Copy(&remoteThing{Name: "thing"}) + + assert.Nil(t, got.ForceSendFields) +} + +func TestCopyDoesNotShareForceSendFieldsWithSource(t *testing.T) { + src := &createThing{ForceSendFields: []string{"Comment", "Size"}} + + got := structcopy.New[createThing, createThing]().Copy(src) + require.Equal(t, []string{"Comment", "Size"}, got.ForceSendFields) + got.ForceSendFields[0] = "Name" + + assert.Equal(t, []string{"Comment", "Size"}, src.ForceSendFields) +} + +func TestCopyIntoEmbeddedStruct(t *testing.T) { + src := &remoteThing{ + Name: "thing", + ThingId: "id-1", + ForceSendFields: []string{"Size", "ThingId"}, + } + + got := structcopy.New[remoteThing, thingState]().Copy(src) + + assert.Equal(t, &thingState{ + Name: "thing", + // ThingId is declared by thingState, which has no ForceSendFields of its own. + ForceSendFields: []string{"Size"}, + ThingId: "id-1", + }, got) +} + +func TestCopyFromEmbeddedStruct(t *testing.T) { + src := &thingState{ + Comment: "a comment", + Tags: map[string]string{"team": "data"}, + ForceSendFields: []string{"Size"}, + ThingId: "id-1", + } + + got := structcopy.New[thingState, updateThing](structcopy.Skip("Name")).Copy(src) + + assert.Equal(t, &updateThing{ + Comment: "a comment", + Tags: map[string]string{"team": "data"}, + ForceSendFields: []string{"Size"}, + }, got) +} + +type stateWithOwnForceSend struct { + createThing + + ThingId string `json:"thing_id,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` +} + +func TestCopyKeepsForceSendFieldsPerStruct(t *testing.T) { + src := &remoteThing{ForceSendFields: []string{"ThingId", "Size"}} + + got := structcopy.New[remoteThing, stateWithOwnForceSend]().Copy(src) + + // A ForceSendFields names fields of the struct that declares it, not of one it embeds. + assert.Equal(t, []string{"Size"}, got.createThing.ForceSendFields) + assert.Equal(t, []string{"ThingId"}, got.ForceSendFields) +} + +type withUnexportedField struct { + Name string `json:"name,omitempty"` + secret string +} + +func TestCopyIgnoresUnexportedFields(t *testing.T) { + src := &withUnexportedField{Name: "thing", secret: "hunter2"} + + got := structcopy.New[withUnexportedField, withUnexportedField]().Copy(src) + + assert.Equal(t, &withUnexportedField{Name: "thing"}, got) +} + +type shadowingState struct { + createThing + + // Name hides createThing.Name, the way a state type overrides a field of the request it embeds. + Name string `json:"name"` +} + +func TestCopyUsesFieldThatShadowsEmbeddedOne(t *testing.T) { + got := structcopy.New[remoteThing, shadowingState]().Copy(&remoteThing{Name: "thing"}) + + assert.Equal(t, "thing", got.Name) + assert.Empty(t, got.createThing.Name) +} + +func TestSkipLeavesFieldUnset(t *testing.T) { + src := &createThing{ + Comment: "a comment", + Name: "thing", + ForceSendFields: []string{"Comment", "Name"}, + } + + got := structcopy.New[createThing, updateThing](structcopy.Skip("Name", "Comment")).Copy(src) + + assert.Empty(t, got.Name) + assert.Empty(t, got.Comment) + // A skipped field carries no value, so there is nothing to force-send. + assert.Nil(t, got.ForceSendFields) +} + +func TestForceSendListsFieldOnEveryCopy(t *testing.T) { + copier := structcopy.New[createThing, updateThing]( + structcopy.Skip("Name"), + structcopy.ForceSend("Comment", "Owner"), + ) + + got := copier.Copy(&createThing{}) + assert.Equal(t, []string{"Comment", "Owner"}, got.ForceSendFields) + + got = copier.Copy(&createThing{ForceSendFields: []string{"Size", "Comment"}}) + assert.Equal(t, []string{"Comment", "Owner", "Size"}, got.ForceSendFields) +} + +func TestForceSendDoesNotAccumulateAcrossCopies(t *testing.T) { + copier := structcopy.New[createThing, updateThing]( + structcopy.Skip("Name"), + structcopy.ForceSend("Comment"), + ) + + copier.Copy(&createThing{ForceSendFields: []string{"Size"}}) + got := copier.Copy(&createThing{}) + + assert.Equal(t, []string{"Comment"}, got.ForceSendFields) +} + +func TestDescribe(t *testing.T) { + copier := structcopy.New[thingState, updateThing]( + structcopy.Skip("Name"), + structcopy.ForceSend("Comment"), + ) + + got, err := copier.Describe() + + require.NoError(t, err) + assert.Equal(t, `structcopy_test.thingState -> structcopy_test.updateThing + Comment: copied, always sent + Name: skipped + NewName: not in source + Owner: not in source + Size: copied + Tags: copied +`, got) +} + +func TestDescribeNamesConvertedType(t *testing.T) { + got, err := structcopy.New[remoteThing, createThing]().Describe() + + require.NoError(t, err) + assert.Contains(t, got, " Kind: copied, converted from structcopy_test.remoteKind\n") +} + +type ( + sizeAsString struct { + Size string `json:"size,omitempty"` + } + sizeRenamedOnWire struct { + Size int `json:"capacity,omitempty"` + } + tagsWithoutForceSend struct { + Tags map[string]string `json:"tags,omitempty"` + } + embeddedPointer struct { + *createThing + } +) + +func TestInvalidCopier(t *testing.T) { + tests := []struct { + name string + describe func() (string, error) + copy func() + want string + }{ + { + name: "source is not a struct", + describe: structcopy.New[string, createThing]().Describe, + copy: func() { structcopy.New[string, createThing]().Copy(new(string)) }, + want: "structcopy: string -> structcopy_test.createThing: string is not a struct", + }, + { + name: "destination is not a struct", + describe: structcopy.New[createThing, int]().Describe, + copy: func() { structcopy.New[createThing, int]().Copy(&createThing{}) }, + want: "structcopy: structcopy_test.createThing -> int: int is not a struct", + }, + { + name: "field has another type in the destination", + describe: structcopy.New[createThing, sizeAsString]().Describe, + copy: func() { structcopy.New[createThing, sizeAsString]().Copy(&createThing{}) }, + want: "structcopy: structcopy_test.createThing -> structcopy_test.sizeAsString: Size is int in the source but string in the destination; skip it and set it by hand", + }, + { + name: "field has another JSON name in the destination", + describe: structcopy.New[createThing, sizeRenamedOnWire]().Describe, + copy: func() { structcopy.New[createThing, sizeRenamedOnWire]().Copy(&createThing{}) }, + want: `structcopy: structcopy_test.createThing -> structcopy_test.sizeRenamedOnWire: Size is "size" in the source's JSON but "capacity" in the destination's; skip it and set it by hand`, + }, + { + name: "skipped field is not in the destination", + describe: structcopy.New[createThing, updateThing](structcopy.Skip("Name", "Kind")).Describe, + copy: func() { structcopy.New[createThing, updateThing](structcopy.Skip("Name", "Kind")).Copy(&createThing{}) }, + want: "structcopy: structcopy_test.createThing -> structcopy_test.updateThing: cannot skip Kind: the destination has no such field", + }, + { + name: "skipped field is not in the source", + describe: structcopy.New[createThing, updateThing](structcopy.Skip("Name", "Owner")).Describe, + copy: func() { + structcopy.New[createThing, updateThing](structcopy.Skip("Name", "Owner")).Copy(&createThing{}) + }, + want: "structcopy: structcopy_test.createThing -> structcopy_test.updateThing: cannot skip Owner: the source has no such field, so it is not copied anyway", + }, + { + name: "force-sent field is not in the destination", + describe: structcopy.New[remoteThing, createThing](structcopy.ForceSend("Owner")).Describe, + copy: func() { structcopy.New[remoteThing, createThing](structcopy.ForceSend("Owner")).Copy(&remoteThing{}) }, + want: "structcopy: structcopy_test.remoteThing -> structcopy_test.createThing: cannot force-send Owner: the destination has no such field", + }, + { + name: "force-sent field is not a basic type", + describe: structcopy.New[remoteThing, createThing](structcopy.ForceSend("Tags")).Describe, + copy: func() { structcopy.New[remoteThing, createThing](structcopy.ForceSend("Tags")).Copy(&remoteThing{}) }, + want: "structcopy: structcopy_test.remoteThing -> structcopy_test.createThing: cannot force-send Tags: ForceSendFields has no effect on a map", + }, + { + name: "force-sent field has no ForceSendFields next to it", + describe: structcopy.New[remoteThing, thingState](structcopy.ForceSend("ThingId")).Describe, + copy: func() { structcopy.New[remoteThing, thingState](structcopy.ForceSend("ThingId")).Copy(&remoteThing{}) }, + want: "structcopy: structcopy_test.remoteThing -> structcopy_test.thingState: cannot force-send ThingId: the struct that declares it has no ForceSendFields", + }, + { + name: "source embeds a pointer", + describe: structcopy.New[embeddedPointer, createThing]().Describe, + copy: func() { structcopy.New[embeddedPointer, createThing]().Copy(&embeddedPointer{}) }, + want: "structcopy: structcopy_test.embeddedPointer -> structcopy_test.createThing: structcopy_test.embeddedPointer embeds *structcopy_test.createThing; embedded pointers are not supported", + }, + { + name: "destination embeds a pointer", + describe: structcopy.New[createThing, embeddedPointer]().Describe, + copy: func() { structcopy.New[createThing, embeddedPointer]().Copy(&createThing{}) }, + want: "structcopy: structcopy_test.createThing -> structcopy_test.embeddedPointer: structcopy_test.embeddedPointer embeds *structcopy_test.createThing; embedded pointers are not supported", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := tt.describe() + require.EqualError(t, err, tt.want) + + assert.PanicsWithError(t, tt.want, tt.copy) + }) + } +} + +func TestSkipAllowsFieldsThatCannotBeCopied(t *testing.T) { + got := structcopy.New[createThing, sizeAsString](structcopy.Skip("Size")).Copy(&createThing{Size: 3}) + + assert.Empty(t, got.Size) +} + +func TestCopyIntoStructWithoutForceSendFields(t *testing.T) { + src := &createThing{ + Tags: map[string]string{"team": "data"}, + ForceSendFields: []string{"Tags"}, + } + + got := structcopy.New[createThing, tagsWithoutForceSend]().Copy(src) + + assert.Equal(t, &tagsWithoutForceSend{Tags: map[string]string{"team": "data"}}, got) +} From 8833b82972f361771f7bb3e58764086f7b431856 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 19 Nov 2026 01:06:46 +0000 Subject: [PATCH 2/9] direct: copy SDK structs with structcopy instead of field by field RemapState and the create, edit and update requests of 20 resources listed every field of the struct they built, so each field the SDK added had to be wired through by hand before exhaustruct passed. They now declare a Copier, which copies the fields both structs have and needs to be told only what to leave out and what to force-send. testdata/field_copies.txt records what every Copier does with each field of its destination. A field the SDK adds changes that file instead of failing the lint, so a bump shows which requests and states pick the field up. A request no longer force-sends the zero value of a field it deliberately leaves unset, which utils.FilterFields allowed when the state listed that field: effective_enable_file_events on an external location update and latest_monitor_failure_msg on a quality monitor update. Requests and remapped state are otherwise unchanged. Co-authored-by: Isaac --- bundle/direct/dresources/catalog.go | 79 +-- bundle/direct/dresources/cluster.go | 145 +--- bundle/direct/dresources/cluster_policy.go | 31 +- bundle/direct/dresources/copy.go | 19 + bundle/direct/dresources/copy_test.go | 39 ++ bundle/direct/dresources/experiment.go | 21 +- bundle/direct/dresources/external_location.go | 76 +-- bundle/direct/dresources/feature.go | 15 +- bundle/direct/dresources/instance_pool.go | 41 +- bundle/direct/dresources/job.go | 42 +- bundle/direct/dresources/model.go | 51 +- .../dresources/model_serving_endpoint.go | 144 ++-- bundle/direct/dresources/pipeline.go | 94 +-- bundle/direct/dresources/quality_monitor.go | 54 +- bundle/direct/dresources/registered_model.go | 75 +-- bundle/direct/dresources/schema.go | 32 +- bundle/direct/dresources/secret.go | 11 +- bundle/direct/dresources/secret_scope.go | 11 +- bundle/direct/dresources/sql_warehouse.go | 53 +- .../dresources/testdata/field_copies.txt | 626 ++++++++++++++++++ .../dresources/vector_search_endpoint.go | 12 +- .../direct/dresources/vector_search_index.go | 40 +- bundle/direct/dresources/volume.go | 47 +- 23 files changed, 958 insertions(+), 800 deletions(-) create mode 100644 bundle/direct/dresources/copy.go create mode 100644 bundle/direct/dresources/copy_test.go create mode 100644 bundle/direct/dresources/testdata/field_copies.txt diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index 53d9b3b23f1..a34860da80a 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -3,10 +3,9 @@ package dresources import ( "context" "fmt" - "slices" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/utils" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -44,21 +43,10 @@ func (*ResourceCatalog) PrepareState(input *resources.Catalog) *CatalogState { } } +var toCatalogState = newCopier[catalog.CatalogInfo, CatalogState]() + func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *CatalogState { - return &CatalogState{ - Comment: info.Comment, - ConnectionName: info.ConnectionName, - CustomMaxRetentionHours: info.CustomMaxRetentionHours, - ManagedEncryptionSettings: info.ManagedEncryptionSettings, - Name: info.Name, - Options: info.Options, - Properties: info.Properties, - ProviderName: info.ProviderName, - ShareName: info.ShareName, - StorageRoot: info.StorageRoot, - ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), - IsolationMode: info.IsolationMode, - } + return toCatalogState.Copy(info) } func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.CatalogInfo, error) { @@ -75,19 +63,12 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s } // UC only accepts isolation_mode on update, so a configured value takes a second call. - response, err = r.client.Catalogs.Update(ctx, catalog.UpdateCatalog{ - Comment: "", - CustomMaxRetentionHours: 0, - EnablePredictiveOptimization: "", - IsolationMode: config.IsolationMode, - ManagedEncryptionSettings: nil, - Name: response.Name, - NewName: "", - Options: nil, - Owner: "", - Properties: nil, - ForceSendFields: nil, - }) + // It carries nothing else: fields are set one by one so that a field the SDK adds to + // UpdateCatalog needs no change here. + var setIsolationMode catalog.UpdateCatalog + setIsolationMode.Name = response.Name + setIsolationMode.IsolationMode = config.IsolationMode + response, err = r.client.Catalogs.Update(ctx, setIsolationMode) if err != nil { // The catalog exists but is not in state, so the next deploy fails with "already // exists"; name the cause here since the caller only knows a create failed. @@ -104,23 +85,18 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s // rejected with "Nothing to update"). var catalogForceSend = []string{"Comment", "CustomMaxRetentionHours"} +// Name addresses the catalog, so it is the id; the name in the config goes in NewName when it differs. +var toUpdateCatalog = newCopier[CatalogState, catalog.UpdateCatalog]( + structcopy.Skip("Name"), + structcopy.ForceSend(catalogForceSend...), +) + // DoUpdate updates the catalog in place and returns remote state. func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (*catalog.CatalogInfo, error) { - updateRequest := catalog.UpdateCatalog{ - Comment: config.Comment, - CustomMaxRetentionHours: config.CustomMaxRetentionHours, - EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: config.IsolationMode, - ManagedEncryptionSettings: config.ManagedEncryptionSettings, - Name: id, - NewName: "", // Only set if name actually changes (see DoUpdateWithID) - Options: config.Options, - Owner: "", // Not supported by DABs - Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "Owner"), - } + updateRequest := toUpdateCatalog.Copy(config) + updateRequest.Name = id - response, err := r.client.Catalogs.Update(ctx, updateRequest) + response, err := r.client.Catalogs.Update(ctx, *updateRequest) if err != nil { return nil, err } @@ -130,25 +106,14 @@ func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *Catal // DoUpdateWithID updates the catalog and returns the new ID if the name changes. func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (string, *catalog.CatalogInfo, error) { - updateRequest := catalog.UpdateCatalog{ - Comment: config.Comment, - CustomMaxRetentionHours: config.CustomMaxRetentionHours, - EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: config.IsolationMode, - ManagedEncryptionSettings: config.ManagedEncryptionSettings, - Name: id, - NewName: "", // Initialized below if needed - Options: config.Options, - Owner: "", // Not supported by DABs - Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "Owner"), - } + updateRequest := toUpdateCatalog.Copy(config) + updateRequest.Name = id if config.Name != id { updateRequest.NewName = config.Name } - response, err := r.client.Catalogs.Update(ctx, updateRequest) + response, err := r.client.Catalogs.Update(ctx, *updateRequest) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index f1e85872cc7..86644741ac3 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" @@ -102,50 +101,12 @@ func (r *ResourceCluster) PrepareState(input *resources.Cluster) *ClusterState { return s } +var toClusterState = newCopier[ClusterRemote, ClusterState]() + // RemapState maps the remote ClusterRemote to ClusterState for diff comparison. // Started is derived from cluster state so the planner can detect start/stop changes. func (r *ResourceCluster) RemapState(input *ClusterRemote) *ClusterState { - spec := &ClusterState{ - ApplyPolicyDefaultValues: input.ApplyPolicyDefaultValues, - Autoscale: input.Autoscale, - AutoterminationMinutes: input.AutoterminationMinutes, - AwsAttributes: input.AwsAttributes, - AzureAttributes: input.AzureAttributes, - ClusterLogConf: input.ClusterLogConf, - ClusterName: input.ClusterName, - CustomTags: input.CustomTags, - DataSecurityMode: input.DataSecurityMode, - DependencyMode: input.DependencyMode, - DockerImage: input.DockerImage, - DriverInstancePoolId: input.DriverInstancePoolId, - DriverNodeTypeId: input.DriverNodeTypeId, - DriverNodeTypeFlexibility: input.DriverNodeTypeFlexibility, - EnableElasticDisk: input.EnableElasticDisk, - EnableLocalDiskEncryption: input.EnableLocalDiskEncryption, - GcpAttributes: input.GcpAttributes, - InitScripts: input.InitScripts, - InstancePoolId: input.InstancePoolId, - IsSingleNode: input.IsSingleNode, - Kind: input.Kind, - NodeTypeId: input.NodeTypeId, - NumWorkers: input.NumWorkers, - PolicyId: input.PolicyId, - RemoteDiskThroughput: input.RemoteDiskThroughput, - RuntimeEngine: input.RuntimeEngine, - SingleUserName: input.SingleUserName, - SparkConf: input.SparkConf, - SparkEnvVars: input.SparkEnvVars, - SparkVersion: input.SparkVersion, - SshPublicKeys: input.SshPublicKeys, - TotalInitialRemoteDiskSize: input.TotalInitialRemoteDiskSize, - UseMlRuntime: input.UseMlRuntime, - WorkloadType: input.WorkloadType, - WorkerNodeTypeFlexibility: input.WorkerNodeTypeFlexibility, - ForceSendFields: utils.FilterFields[compute.ClusterSpec](input.ForceSendFields), - Lifecycle: input.Lifecycle, - Libraries: input.Libraries, - } - return spec + return toClusterState.Copy(input) } func (r *ResourceCluster) DoRead(ctx context.Context, id string) (*ClusterRemote, error) { @@ -349,12 +310,9 @@ func (r *ResourceCluster) WaitAfterCreate(ctx context.Context, id string, config } func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *ClusterState, entry *PlanEntry) error { - _, err := r.client.Clusters.Resize(ctx, compute.ResizeCluster{ - ClusterId: id, - NumWorkers: config.NumWorkers, - Autoscale: config.Autoscale, - ForceSendFields: utils.FilterFields[compute.ResizeCluster](config.ForceSendFields), - }) + resize := toResizeCluster.Copy(config) + resize.ClusterId = id + _, err := r.client.Clusters.Resize(ctx, *resize) if err == nil { return nil } @@ -427,46 +385,14 @@ func remoteClusterIsRunning(entry *PlanEntry) bool { return remote.State == compute.StateRunning } +var ( + toCreateCluster = newCopier[compute.ClusterSpec, compute.CreateCluster]() + toEditCluster = newCopier[compute.ClusterSpec, compute.EditCluster]() + toResizeCluster = newCopier[ClusterState, compute.ResizeCluster]() +) + func makeCreateCluster(config *compute.ClusterSpec) compute.CreateCluster { - create := compute.CreateCluster{ - ApplyPolicyDefaultValues: config.ApplyPolicyDefaultValues, - Autoscale: config.Autoscale, - AutoterminationMinutes: config.AutoterminationMinutes, - AwsAttributes: config.AwsAttributes, - AzureAttributes: config.AzureAttributes, - ClusterLogConf: config.ClusterLogConf, - ClusterName: config.ClusterName, - CloneFrom: nil, // Not supported by DABs - CustomTags: config.CustomTags, - DataSecurityMode: config.DataSecurityMode, - DependencyMode: config.DependencyMode, - DockerImage: config.DockerImage, - DriverInstancePoolId: config.DriverInstancePoolId, - DriverNodeTypeId: config.DriverNodeTypeId, - DriverNodeTypeFlexibility: config.DriverNodeTypeFlexibility, - EnableElasticDisk: config.EnableElasticDisk, - EnableLocalDiskEncryption: config.EnableLocalDiskEncryption, - GcpAttributes: config.GcpAttributes, - InitScripts: config.InitScripts, - InstancePoolId: config.InstancePoolId, - IsSingleNode: config.IsSingleNode, - Kind: config.Kind, - NodeTypeId: config.NodeTypeId, - NumWorkers: config.NumWorkers, - PolicyId: config.PolicyId, - RemoteDiskThroughput: config.RemoteDiskThroughput, - RuntimeEngine: config.RuntimeEngine, - SingleUserName: config.SingleUserName, - SparkConf: config.SparkConf, - SparkEnvVars: config.SparkEnvVars, - SparkVersion: config.SparkVersion, - SshPublicKeys: config.SshPublicKeys, - TotalInitialRemoteDiskSize: config.TotalInitialRemoteDiskSize, - UseMlRuntime: config.UseMlRuntime, - WorkloadType: config.WorkloadType, - WorkerNodeTypeFlexibility: config.WorkerNodeTypeFlexibility, - ForceSendFields: utils.FilterFields[compute.CreateCluster](config.ForceSendFields), - } + create := toCreateCluster.Copy(config) // If autoscale is not set, we need to send NumWorkers because one of them is required. // If NumWorkers is not nil, we don't need to set it to ForceSendFields as it will be sent anyway. @@ -474,49 +400,12 @@ func makeCreateCluster(config *compute.ClusterSpec) compute.CreateCluster { create.ForceSendFields = append(create.ForceSendFields, "NumWorkers") } - return create + return *create } func makeEditCluster(id string, config *compute.ClusterSpec) compute.EditCluster { - edit := compute.EditCluster{ - ClusterId: id, - ApplyPolicyDefaultValues: config.ApplyPolicyDefaultValues, - Autoscale: config.Autoscale, - AutoterminationMinutes: config.AutoterminationMinutes, - AwsAttributes: config.AwsAttributes, - AzureAttributes: config.AzureAttributes, - ClusterLogConf: config.ClusterLogConf, - ClusterName: config.ClusterName, - CustomTags: config.CustomTags, - DataSecurityMode: config.DataSecurityMode, - DependencyMode: config.DependencyMode, - DockerImage: config.DockerImage, - DriverInstancePoolId: config.DriverInstancePoolId, - DriverNodeTypeId: config.DriverNodeTypeId, - DriverNodeTypeFlexibility: config.DriverNodeTypeFlexibility, - EnableElasticDisk: config.EnableElasticDisk, - EnableLocalDiskEncryption: config.EnableLocalDiskEncryption, - GcpAttributes: config.GcpAttributes, - InitScripts: config.InitScripts, - InstancePoolId: config.InstancePoolId, - IsSingleNode: config.IsSingleNode, - Kind: config.Kind, - NodeTypeId: config.NodeTypeId, - NumWorkers: config.NumWorkers, - PolicyId: config.PolicyId, - RemoteDiskThroughput: config.RemoteDiskThroughput, - RuntimeEngine: config.RuntimeEngine, - SingleUserName: config.SingleUserName, - SparkConf: config.SparkConf, - SparkEnvVars: config.SparkEnvVars, - SparkVersion: config.SparkVersion, - SshPublicKeys: config.SshPublicKeys, - TotalInitialRemoteDiskSize: config.TotalInitialRemoteDiskSize, - UseMlRuntime: config.UseMlRuntime, - WorkloadType: config.WorkloadType, - WorkerNodeTypeFlexibility: config.WorkerNodeTypeFlexibility, - ForceSendFields: utils.FilterFields[compute.EditCluster](config.ForceSendFields), - } + edit := toEditCluster.Copy(config) + edit.ClusterId = id // If autoscale is not set, we need to send NumWorkers because one of them is required. // If NumWorkers is not nil, we don't need to set it to ForceSendFields as it will be sent anyway. @@ -524,7 +413,7 @@ func makeEditCluster(id string, config *compute.ClusterSpec) compute.EditCluster edit.ForceSendFields = append(edit.ForceSendFields, "NumWorkers") } - return edit + return *edit } // librariesPath is the ClusterState path of the libraries slice, used to detect library changes. diff --git a/bundle/direct/dresources/cluster_policy.go b/bundle/direct/dresources/cluster_policy.go index ae69390b9cc..b6ba7328fca 100644 --- a/bundle/direct/dresources/cluster_policy.go +++ b/bundle/direct/dresources/cluster_policy.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/compute" ) @@ -29,19 +28,15 @@ func (*ResourceClusterPolicy) PrepareState(input *resources.ClusterPolicy) *comp return &cp } +var ( + toCreatePolicy = newCopier[compute.Policy, compute.CreatePolicy]() + toEditPolicy = newCopier[compute.CreatePolicy, compute.EditPolicy]() +) + // RemapState copies the config fields shared by Policy and CreatePolicy; // output-only fields (policy_id, created_at_timestamp, creator_user_name, is_default) are not in the state. func (*ResourceClusterPolicy) RemapState(remote *compute.Policy) *compute.CreatePolicy { - return &compute.CreatePolicy{ - Definition: remote.Definition, - Description: remote.Description, - Libraries: remote.Libraries, - MaxClustersPerUser: remote.MaxClustersPerUser, - Name: remote.Name, - PolicyFamilyDefinitionOverrides: remote.PolicyFamilyDefinitionOverrides, - PolicyFamilyId: remote.PolicyFamilyId, - ForceSendFields: utils.FilterFields[compute.CreatePolicy](remote.ForceSendFields), - } + return toCreatePolicy.Copy(remote) } func (r *ResourceClusterPolicy) DoRead(ctx context.Context, id string) (*compute.Policy, error) { @@ -58,17 +53,9 @@ func (r *ResourceClusterPolicy) DoCreate(ctx context.Context, config *compute.Cr } func (r *ResourceClusterPolicy) DoUpdate(ctx context.Context, id string, config *compute.CreatePolicy, _ *PlanEntry) (*compute.Policy, error) { - return nil, r.client.ClusterPolicies.Edit(ctx, compute.EditPolicy{ - PolicyId: id, - Name: config.Name, - Definition: config.Definition, - Description: config.Description, - Libraries: config.Libraries, - MaxClustersPerUser: config.MaxClustersPerUser, - PolicyFamilyDefinitionOverrides: config.PolicyFamilyDefinitionOverrides, - PolicyFamilyId: config.PolicyFamilyId, - ForceSendFields: utils.FilterFields[compute.EditPolicy](config.ForceSendFields), - }) + edit := toEditPolicy.Copy(config) + edit.PolicyId = id + return nil, r.client.ClusterPolicies.Edit(ctx, *edit) } func (r *ResourceClusterPolicy) DoDelete(ctx context.Context, id string, _ *compute.CreatePolicy) error { diff --git a/bundle/direct/dresources/copy.go b/bundle/direct/dresources/copy.go new file mode 100644 index 00000000000..8e4a9a0db35 --- /dev/null +++ b/bundle/direct/dresources/copy.go @@ -0,0 +1,19 @@ +package dresources + +import "github.com/databricks/cli/libs/structs/structcopy" + +// fieldCopier is the part of a structcopy.Copier that does not depend on the copied types. +type fieldCopier interface { + Describe() (string, error) +} + +// fieldCopiers holds every Copier of this package, for TestFieldCopies. +var fieldCopiers []fieldCopier + +// newCopier returns a Copier for the fields that Src and Dst share; see structcopy.New. +// Declare it in a package-level variable, so that TestFieldCopies covers it. +func newCopier[Src, Dst any](opts ...structcopy.Option) *structcopy.Copier[Src, Dst] { + copier := structcopy.New[Src, Dst](opts...) + fieldCopiers = append(fieldCopiers, copier) + return copier +} diff --git a/bundle/direct/dresources/copy_test.go b/bundle/direct/dresources/copy_test.go new file mode 100644 index 00000000000..eab932af631 --- /dev/null +++ b/bundle/direct/dresources/copy_test.go @@ -0,0 +1,39 @@ +package dresources + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/databricks/cli/libs/testdiff" + "github.com/stretchr/testify/require" +) + +// TestFieldCopies checks every Copier of this package and compares what they copy with +// testdata/field_copies.txt. An SDK bump that adds a field changes the file, so the bump +// shows which requests and states pick the field up; `./task generate-direct` updates it. +func TestFieldCopies(t *testing.T) { + require.NotEmpty(t, fieldCopiers) + + descriptions := make([]string, 0, len(fieldCopiers)) + for _, copier := range fieldCopiers { + description, err := copier.Describe() + require.NoError(t, err) + descriptions = append(descriptions, description) + } + slices.Sort(descriptions) + got := strings.Join(descriptions, "\n") + + path := filepath.Join("testdata", "field_copies.txt") + if testdiff.OverwriteMode { + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte(got), 0o644)) + return + } + + want, err := os.ReadFile(path) + require.NoError(t, err) + testdiff.AssertEqualTexts(t, path, "actual", testdiff.NormalizeNewlines(string(want)), got) +} diff --git a/bundle/direct/dresources/experiment.go b/bundle/direct/dresources/experiment.go index e4f2e8ebbd7..539fba6d0fe 100644 --- a/bundle/direct/dresources/experiment.go +++ b/bundle/direct/dresources/experiment.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -23,23 +22,13 @@ func (*ResourceExperiment) New(client *databricks.WorkspaceClient) *ResourceExpe } func (*ResourceExperiment) PrepareState(input *resources.MlflowExperiment) *ml.CreateExperiment { - return &ml.CreateExperiment{ - Name: input.Name, - ArtifactLocation: input.ArtifactLocation, - Tags: input.Tags, - TraceLocation: input.TraceLocation, - ForceSendFields: utils.FilterFields[ml.CreateExperiment](input.ForceSendFields), - } + return &input.CreateExperiment } +var toCreateExperiment = newCopier[ml.Experiment, ml.CreateExperiment]() + func (*ResourceExperiment) RemapState(experiment *ml.Experiment) *ml.CreateExperiment { - return &ml.CreateExperiment{ - Name: experiment.Name, - ArtifactLocation: experiment.ArtifactLocation, - Tags: experiment.Tags, - TraceLocation: experiment.TraceLocation, - ForceSendFields: utils.FilterFields[ml.CreateExperiment](experiment.ForceSendFields), - } + return toCreateExperiment.Copy(experiment) } func (r *ResourceExperiment) DoRead(ctx context.Context, id string) (*ml.Experiment, error) { @@ -64,7 +53,7 @@ func (r *ResourceExperiment) DoUpdate(ctx context.Context, id string, config *ml updateReq := ml.UpdateExperiment{ ExperimentId: id, NewName: config.Name, - ForceSendFields: utils.FilterFields[ml.UpdateExperiment](config.ForceSendFields), + ForceSendFields: nil, } return nil, r.client.Experiments.UpdateExperiment(ctx, updateReq) diff --git a/bundle/direct/dresources/external_location.go b/bundle/direct/dresources/external_location.go index 26c957020ab..c3b00645959 100644 --- a/bundle/direct/dresources/external_location.go +++ b/bundle/direct/dresources/external_location.go @@ -2,10 +2,9 @@ package dresources import ( "context" - "slices" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/utils" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -22,23 +21,11 @@ func (*ResourceExternalLocation) PrepareState(input *resources.ExternalLocation) return &input.CreateExternalLocation } +// The output-only effective_* fields are mirrored into state to avoid churn in remapped config. +var toCreateExternalLocation = newCopier[catalog.ExternalLocationInfo, catalog.CreateExternalLocation]() + func (*ResourceExternalLocation) RemapState(info *catalog.ExternalLocationInfo) *catalog.CreateExternalLocation { - return &catalog.CreateExternalLocation{ - Comment: info.Comment, - CredentialName: info.CredentialName, - // Output-only fields mirrored into state to avoid churn in remapped config. - EffectiveEnableFileEvents: info.EffectiveEnableFileEvents, - EffectiveFileEventQueue: info.EffectiveFileEventQueue, - EnableFileEvents: info.EnableFileEvents, - EncryptionDetails: info.EncryptionDetails, - Fallback: info.Fallback, - FileEventQueue: info.FileEventQueue, - Name: info.Name, - ReadOnly: info.ReadOnly, - SkipValidation: false, // This is an input-only parameter, never returned by API - Url: info.Url, - ForceSendFields: utils.FilterFields[catalog.CreateExternalLocation](info.ForceSendFields), - } + return toCreateExternalLocation.Copy(info) } func (r *ResourceExternalLocation) DoRead(ctx context.Context, id string) (*catalog.ExternalLocationInfo, error) { @@ -60,58 +47,31 @@ func (r *ResourceExternalLocation) DoCreate(ctx context.Context, config *catalog // {"comment": ""}; re-probe before adding anything else. var externalLocationForceSend = []string{"Comment"} +// The effective_* fields are output-only, so they are never sent in an update. Name addresses +// the location, so it is the id; the name in the config goes in NewName when it differs. +var toUpdateExternalLocation = newCopier[catalog.CreateExternalLocation, catalog.UpdateExternalLocation]( + structcopy.Skip("EffectiveEnableFileEvents", "EffectiveFileEventQueue", "Name"), + structcopy.ForceSend(externalLocationForceSend...), +) + // DoUpdate updates the external location in place and returns remote state. func (r *ResourceExternalLocation) DoUpdate(ctx context.Context, id string, config *catalog.CreateExternalLocation, _ *PlanEntry) (*catalog.ExternalLocationInfo, error) { - updateRequest := catalog.UpdateExternalLocation{ - Comment: config.Comment, - CredentialName: config.CredentialName, - EffectiveEnableFileEvents: false, // Output-only field; never sent in update payload. - EffectiveFileEventQueue: nil, - EnableFileEvents: config.EnableFileEvents, - EncryptionDetails: config.EncryptionDetails, - Fallback: config.Fallback, - FileEventQueue: config.FileEventQueue, - Force: false, - IsolationMode: "", // Not supported by DABs - Name: id, - NewName: "", // Only set if name actually changes (see DoUpdateWithID) - Owner: "", // Not supported by DABs - ReadOnly: config.ReadOnly, - SkipValidation: config.SkipValidation, - Url: config.Url, - ForceSendFields: utils.FilterFields[catalog.UpdateExternalLocation](append(slices.Clone(externalLocationForceSend), config.ForceSendFields...), "IsolationMode", "Owner"), - } + updateRequest := toUpdateExternalLocation.Copy(config) + updateRequest.Name = id - return r.client.ExternalLocations.Update(ctx, updateRequest) + return r.client.ExternalLocations.Update(ctx, *updateRequest) } // DoUpdateWithID updates the external location and returns the new ID if the name changes. func (r *ResourceExternalLocation) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateExternalLocation, _ *PlanEntry) (string, *catalog.ExternalLocationInfo, error) { - updateRequest := catalog.UpdateExternalLocation{ - Comment: config.Comment, - CredentialName: config.CredentialName, - EffectiveEnableFileEvents: false, // Output-only field; never sent in update payload. - EffectiveFileEventQueue: nil, - EnableFileEvents: config.EnableFileEvents, - EncryptionDetails: config.EncryptionDetails, - Fallback: config.Fallback, - FileEventQueue: config.FileEventQueue, - Force: false, - IsolationMode: "", // Not supported by DABs - Name: id, - NewName: "", // Initialized below if needed - Owner: "", // Not supported by DABs - ReadOnly: config.ReadOnly, - SkipValidation: config.SkipValidation, - Url: config.Url, - ForceSendFields: utils.FilterFields[catalog.UpdateExternalLocation](append(slices.Clone(externalLocationForceSend), config.ForceSendFields...), "IsolationMode", "Owner"), - } + updateRequest := toUpdateExternalLocation.Copy(config) + updateRequest.Name = id if config.Name != id { updateRequest.NewName = config.Name } - response, err := r.client.ExternalLocations.Update(ctx, updateRequest) + response, err := r.client.ExternalLocations.Update(ctx, *updateRequest) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/feature.go b/bundle/direct/dresources/feature.go index c5c8265bff4..3aadccc7539 100644 --- a/bundle/direct/dresources/feature.go +++ b/bundle/direct/dresources/feature.go @@ -2,11 +2,11 @@ package dresources import ( "context" - "slices" "strings" "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -59,17 +59,18 @@ func (r *ResourceFeature) DoCreate(ctx context.Context, config *ml.Feature) (str // field is excluded at plan level in configs/features.yml and features.generated.yml. const featureUpdateMask = "description" +// The whole feature goes in the body, with the mask limiting what is applied: full_name, +// function and source have no omitempty, so a body built from just the updatable fields would +// send them blanked out. Description is forced so clearing it sends "" rather than omitting it. +var toUpdateFeature = newCopier[ml.Feature, ml.Feature](structcopy.ForceSend("Description")) + func (r *ResourceFeature) DoUpdate(ctx context.Context, id string, config *ml.Feature, _ *PlanEntry) (*ml.Feature, error) { - // The whole feature goes in the body, with the mask limiting what is applied: full_name, - // function and source have no omitempty, so a body built from just the updatable fields would - // send them blanked out. Description is forced so clearing it sends "" rather than omitting it. - update := *config + update := toUpdateFeature.Copy(config) update.FullName = id - update.ForceSendFields = append(slices.Clone(config.ForceSendFields), "Description") return r.client.FeatureEngineering.UpdateFeature(ctx, ml.UpdateFeatureRequest{ FullName: id, - Feature: update, + Feature: *update, UpdateMask: featureUpdateMask, }) } diff --git a/bundle/direct/dresources/instance_pool.go b/bundle/direct/dresources/instance_pool.go index 37a1aeab447..08a48a4851b 100644 --- a/bundle/direct/dresources/instance_pool.go +++ b/bundle/direct/dresources/instance_pool.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/compute" ) @@ -21,28 +20,15 @@ func (*ResourceInstancePool) PrepareState(input *resources.InstancePool) *comput return &input.CreateInstancePool } +var ( + toCreateInstancePool = newCopier[compute.GetInstancePool, compute.CreateInstancePool]() + toEditInstancePool = newCopier[compute.CreateInstancePool, compute.EditInstancePool]() +) + // RemapState copies the config fields shared by GetInstancePool and CreateInstancePool; // output-only fields (state, stats, default_tags, instance_pool_id) are not in the state. func (*ResourceInstancePool) RemapState(remote *compute.GetInstancePool) *compute.CreateInstancePool { - return &compute.CreateInstancePool{ - AwsAttributes: remote.AwsAttributes, - AzureAttributes: remote.AzureAttributes, - CustomTags: remote.CustomTags, - DiskSpec: remote.DiskSpec, - EnableElasticDisk: remote.EnableElasticDisk, - GcpAttributes: remote.GcpAttributes, - IdleInstanceAutoterminationMinutes: remote.IdleInstanceAutoterminationMinutes, - InstancePoolName: remote.InstancePoolName, - MaxCapacity: remote.MaxCapacity, - MinIdleInstances: remote.MinIdleInstances, - NodeTypeFlexibility: remote.NodeTypeFlexibility, - NodeTypeId: remote.NodeTypeId, - PreloadedDockerImages: remote.PreloadedDockerImages, - PreloadedSparkVersions: remote.PreloadedSparkVersions, - RemoteDiskThroughput: remote.RemoteDiskThroughput, - TotalInitialRemoteDiskSize: remote.TotalInitialRemoteDiskSize, - ForceSendFields: utils.FilterFields[compute.CreateInstancePool](remote.ForceSendFields), - } + return toCreateInstancePool.Copy(remote) } func (r *ResourceInstancePool) DoRead(ctx context.Context, id string) (*compute.GetInstancePool, error) { @@ -58,18 +44,9 @@ func (r *ResourceInstancePool) DoCreate(ctx context.Context, config *compute.Cre } func (r *ResourceInstancePool) DoUpdate(ctx context.Context, id string, config *compute.CreateInstancePool, _ *PlanEntry) (*compute.GetInstancePool, error) { - return nil, r.client.InstancePools.Edit(ctx, compute.EditInstancePool{ - InstancePoolId: id, - InstancePoolName: config.InstancePoolName, - NodeTypeId: config.NodeTypeId, - MinIdleInstances: config.MinIdleInstances, - MaxCapacity: config.MaxCapacity, - IdleInstanceAutoterminationMinutes: config.IdleInstanceAutoterminationMinutes, - CustomTags: config.CustomTags, - RemoteDiskThroughput: config.RemoteDiskThroughput, - TotalInitialRemoteDiskSize: config.TotalInitialRemoteDiskSize, - ForceSendFields: utils.FilterFields[compute.EditInstancePool](config.ForceSendFields), - }) + edit := toEditInstancePool.Copy(config) + edit.InstancePoolId = id + return nil, r.client.InstancePools.Edit(ctx, *edit) } func (r *ResourceInstancePool) DoDelete(ctx context.Context, id string, _ *compute.CreateInstancePool) error { diff --git a/bundle/direct/dresources/job.go b/bundle/direct/dresources/job.go index a19ecf4dae2..e0e7c3947fa 100644 --- a/bundle/direct/dresources/job.go +++ b/bundle/direct/dresources/job.go @@ -6,7 +6,6 @@ import ( "strconv" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -179,44 +178,11 @@ func (r *ResourceJob) DoDelete(ctx context.Context, id string, _ *jobs.JobSettin return r.client.Jobs.DeleteByJobId(ctx, idInt) } -func makeCreateJob(config jobs.JobSettings) (jobs.CreateJob, error) { - // Note, exhaustruct linter validates that all off CreateJob fields are initialized. - // We don't have linter that validates that all of config fields are used. - result := jobs.CreateJob{ - AccessControlList: nil, // Not supported by DABs - BudgetPolicyId: config.BudgetPolicyId, - Continuous: config.Continuous, - Deployment: config.Deployment, - Description: config.Description, - DurationTarget: config.DurationTarget, - EditMode: config.EditMode, - EmailNotifications: config.EmailNotifications, - EnvironmentVariables: config.EnvironmentVariables, - Environments: config.Environments, - Format: config.Format, - GitSource: config.GitSource, - Health: config.Health, - JobClusters: config.JobClusters, - MaxConcurrentRuns: config.MaxConcurrentRuns, - Name: config.Name, - NotificationSettings: config.NotificationSettings, - Parameters: config.Parameters, - ParentPath: config.ParentPath, - PerformanceTarget: config.PerformanceTarget, - Queue: config.Queue, - RunAs: config.RunAs, - Schedule: config.Schedule, - Tags: config.Tags, - Tasks: config.Tasks, - TimeoutSeconds: config.TimeoutSeconds, - Trigger: config.Trigger, - Triggers: config.Triggers, - UsagePolicyId: config.UsagePolicyId, - WebhookNotifications: config.WebhookNotifications, - ForceSendFields: utils.FilterFields[jobs.CreateJob](config.ForceSendFields, "AccessControlList"), - } +// AccessControlList stays empty: it is not supported by DABs. +var toCreateJob = newCopier[jobs.JobSettings, jobs.CreateJob]() - return result, nil +func makeCreateJob(config jobs.JobSettings) (jobs.CreateJob, error) { + return *toCreateJob.Copy(&config), nil } func makeResetJob(config jobs.JobSettings, id string) (jobs.ResetJob, error) { diff --git a/bundle/direct/dresources/model.go b/bundle/direct/dresources/model.go index 83a4f8832aa..4442260d808 100644 --- a/bundle/direct/dresources/model.go +++ b/bundle/direct/dresources/model.go @@ -5,7 +5,7 @@ import ( "errors" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/utils" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/ml" @@ -42,13 +42,19 @@ func (*ResourceMlflowModel) PrepareState(input *resources.MlflowModel) *ml.Creat return &input.CreateModelRequest } +var ( + toCreateModel = newCopier[MlflowModelRemote, ml.CreateModelRequest]() + + // Name addresses the model, so it is the id. + toUpdateModel = newCopier[ml.CreateModelRequest, ml.UpdateModelRequest](structcopy.Skip("Name")) + + // ml.Model, the UpdateModel response, carries neither Id nor PermissionLevel, so they + // stay empty; ModelId is carried forward from the remote state the update started from. + toMlflowModelRemote = newCopier[ml.Model, MlflowModelRemote]() +) + func (*ResourceMlflowModel) RemapState(output *MlflowModelRemote) *ml.CreateModelRequest { - return &ml.CreateModelRequest{ - Name: output.Name, - Tags: output.Tags, - Description: output.Description, - ForceSendFields: utils.FilterFields[ml.CreateModelRequest](output.ForceSendFields), - } + return toCreateModel.Copy(output) } func (r *ResourceMlflowModel) DoRead(ctx context.Context, id string) (*MlflowModelRemote, error) { @@ -78,13 +84,10 @@ func (r *ResourceMlflowModel) DoCreate(ctx context.Context, config *ml.CreateMod } func (r *ResourceMlflowModel) DoUpdate(ctx context.Context, id string, config *ml.CreateModelRequest, entry *PlanEntry) (*MlflowModelRemote, error) { - updateRequest := ml.UpdateModelRequest{ - Name: id, - Description: config.Description, - ForceSendFields: utils.FilterFields[ml.UpdateModelRequest](config.ForceSendFields), - } + updateRequest := toUpdateModel.Copy(config) + updateRequest.Name = id - response, err := r.client.ModelRegistry.UpdateModel(ctx, updateRequest) + response, err := r.client.ModelRegistry.UpdateModel(ctx, *updateRequest) if err != nil { return nil, err } @@ -98,23 +101,11 @@ func (r *ResourceMlflowModel) DoUpdate(ctx context.Context, id string, config *m modelId = old.ModelId } - // Id and PermissionLevel are left empty because ml.Model (the UpdateModel - // response) does not carry them; every other field is taken from the - // response so references against the post-update remote state resolve to - // real values until the next plan's DoRead. - return &MlflowModelRemote{ - CreationTimestamp: response.RegisteredModel.CreationTimestamp, - Description: response.RegisteredModel.Description, - Id: "", - LastUpdatedTimestamp: response.RegisteredModel.LastUpdatedTimestamp, - LatestVersions: response.RegisteredModel.LatestVersions, - Name: response.RegisteredModel.Name, - PermissionLevel: "", - Tags: response.RegisteredModel.Tags, - UserId: response.RegisteredModel.UserId, - ForceSendFields: utils.FilterFields[ml.ModelDatabricks](response.RegisteredModel.ForceSendFields, "Id", "PermissionLevel"), - ModelId: modelId, - }, nil + // Every field the response has is taken from it, so references against the + // post-update remote state resolve to real values until the next plan's DoRead. + remote := toMlflowModelRemote.Copy(response.RegisteredModel) + remote.ModelId = modelId + return remote, nil } func (r *ResourceMlflowModel) DoDelete(ctx context.Context, id string, _ *ml.CreateModelRequest) error { diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index 26f181ab205..81a4fbcd2b2 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -6,8 +6,8 @@ import ( "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/serving" @@ -36,6 +36,20 @@ func (*ResourceModelServingEndpoint) PrepareState(input *resources.ModelServingE return &input.CreateServingEndpoint } +// The API reports a config with types of its own (the *Output ones); these convert them +// to the types a request takes. +var ( + //nolint:staticcheck // SA1019: deprecated AutoCaptureConfig{Input,Output} kept for bundle config compatibility + toAutoCaptureConfigInput = newCopier[serving.AutoCaptureConfigOutput, serving.AutoCaptureConfigInput]() + toServedEntityInput = newCopier[serving.ServedEntityOutput, serving.ServedEntityInput]() + + // AutoCaptureConfig and ServedEntities are converted on their own. ServedModels, which + // served_entities deprecates, is left unset. + toEndpointCoreConfigInput = newCopier[serving.EndpointCoreConfigOutput, serving.EndpointCoreConfigInput]( + structcopy.Skip("AutoCaptureConfig", "ServedEntities", "ServedModels"), + ) +) + // AutoCaptureConfig is the legacy inference-table API; the recommended replacement // is AI Gateway inference tables. Bundles still expose it, so the conversion has to // keep working until users have migrated. @@ -45,36 +59,13 @@ func autoCaptureConfigOutputToInput(output *serving.AutoCaptureConfigOutput) *se if output == nil { return nil } - return &serving.AutoCaptureConfigInput{ - CatalogName: output.CatalogName, - SchemaName: output.SchemaName, - TableNamePrefix: output.TableNamePrefix, - Enabled: output.Enabled, - ForceSendFields: utils.FilterFields[serving.AutoCaptureConfigInput](output.ForceSendFields), - } + return toAutoCaptureConfigInput.Copy(output) } func servedEntitiesOutputToInput(output []serving.ServedEntityOutput) []serving.ServedEntityInput { entities := make([]serving.ServedEntityInput, len(output)) - for i, entity := range output { - entities[i] = serving.ServedEntityInput{ - EntityName: entity.EntityName, - EntityVersion: entity.EntityVersion, - EnvironmentVars: entity.EnvironmentVars, - ExternalModel: entity.ExternalModel, - InstanceProfileArn: entity.InstanceProfileArn, - MaxProvisionedConcurrency: entity.MaxProvisionedConcurrency, - MaxProvisionedThroughput: entity.MaxProvisionedThroughput, - MinProvisionedConcurrency: entity.MinProvisionedConcurrency, - MinProvisionedThroughput: entity.MinProvisionedThroughput, - Name: entity.Name, - ProvisionedModelUnits: entity.ProvisionedModelUnits, - ScaleToZeroEnabled: entity.ScaleToZeroEnabled, - WorkloadSize: entity.WorkloadSize, - WorkloadType: entity.WorkloadType, - BurstScalingEnabled: entity.BurstScalingEnabled, - ForceSendFields: utils.FilterFields[serving.ServedEntityInput](entity.ForceSendFields), - } + for i := range output { + entities[i] = *toServedEntityInput.Copy(&output[i]) } return entities @@ -84,32 +75,19 @@ func configOutputToInput(output *serving.EndpointCoreConfigOutput) *serving.Endp if output == nil { return nil } - return &serving.EndpointCoreConfigInput{ - AutoCaptureConfig: autoCaptureConfigOutputToInput(output.AutoCaptureConfig), - ServedEntities: servedEntitiesOutputToInput(output.ServedEntities), - ServedModels: nil, - TrafficConfig: output.TrafficConfig, - Name: "", - } + config := toEndpointCoreConfigInput.Copy(output) + config.AutoCaptureConfig = autoCaptureConfigOutputToInput(output.AutoCaptureConfig) + config.ServedEntities = servedEntitiesOutputToInput(output.ServedEntities) + return config } +// Rate limits are a deprecated field that are not returned by the API on GET calls, so the +// remote state has no value for them. +// TODO(shreyas): Add a warning when users try setting top level rate limits. +var toCreateServingEndpoint = newCopier[ModelServingEndpointRemote, serving.CreateServingEndpoint]() + func (*ResourceModelServingEndpoint) RemapState(state *ModelServingEndpointRemote) *serving.CreateServingEndpoint { - return &serving.CreateServingEndpoint{ - AiGateway: state.AiGateway, - BudgetPolicyId: state.BudgetPolicyId, - Config: state.Config, - Description: state.Description, - EmailNotifications: state.EmailNotifications, - Name: state.Name, - RouteOptimized: state.RouteOptimized, - Tags: state.Tags, - TelemetryConfig: state.TelemetryConfig, - ForceSendFields: utils.FilterFields[serving.CreateServingEndpoint](state.ForceSendFields), - - // Rate limits are a deprecated field that are not returned by the API on GET calls. Thus we map them to nil. - // TODO(shreyas): Add a warning when users try setting top level rate limits. - RateLimits: nil, - } + return toCreateServingEndpoint.Copy(state) } type ModelServingEndpointRemote struct { @@ -143,21 +121,17 @@ func (s ModelServingEndpointRemote) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } +// Config is converted on its own, see configOutputToInput. +var toModelServingEndpointRemote = newCopier[serving.ServingEndpointDetailed, ModelServingEndpointRemote]( + structcopy.Skip("Config"), +) + func newModelServingEndpointRemote(details *serving.ServingEndpointDetailed) *ModelServingEndpointRemote { - return &ModelServingEndpointRemote{ - EndpointDetails: details, - EndpointId: details.Id, - AiGateway: details.AiGateway, - BudgetPolicyId: details.BudgetPolicyId, - Config: configOutputToInput(details.Config), - Description: details.Description, - EmailNotifications: details.EmailNotifications, - Name: details.Name, - RouteOptimized: details.RouteOptimized, - Tags: details.Tags, - TelemetryConfig: details.TelemetryConfig, - ForceSendFields: utils.FilterFields[ModelServingEndpointRemote](details.ForceSendFields), - } + remote := toModelServingEndpointRemote.Copy(details) + remote.EndpointDetails = details + remote.EndpointId = details.Id + remote.Config = configOutputToInput(details.Config) + return remote } func (r *ResourceModelServingEndpoint) DoRead(ctx context.Context, id string) (*ModelServingEndpointRemote, error) { @@ -217,29 +191,19 @@ func (r *ResourceModelServingEndpoint) WaitAfterUpdate(ctx context.Context, id s return r.waitForEndpointReady(ctx, config.Name) } +var toPutAiGateway = newCopier[serving.AiGatewayConfig, serving.PutAiGatewayRequest]() + func (r *ResourceModelServingEndpoint) updateAiGateway(ctx context.Context, id string, aiGateway *serving.AiGatewayConfig) error { if aiGateway == nil { - req := serving.PutAiGatewayRequest{ - Name: id, - FallbackConfig: nil, - Guardrails: nil, - InferenceTableConfig: nil, - RateLimits: nil, - UsageTrackingConfig: nil, - } + var req serving.PutAiGatewayRequest + req.Name = id _, err := r.client.ServingEndpoints.PutAiGateway(ctx, req) return err } - req := serving.PutAiGatewayRequest{ - Name: id, - FallbackConfig: aiGateway.FallbackConfig, - Guardrails: aiGateway.Guardrails, - InferenceTableConfig: aiGateway.InferenceTableConfig, - RateLimits: aiGateway.RateLimits, - UsageTrackingConfig: aiGateway.UsageTrackingConfig, - } - _, err := r.client.ServingEndpoints.PutAiGateway(ctx, req) + req := toPutAiGateway.Copy(aiGateway) + req.Name = id + _, err := r.client.ServingEndpoints.PutAiGateway(ctx, *req) if err != nil { return fmt.Errorf("failed to update AI gateway: %w", err) } @@ -250,23 +214,13 @@ func (r *ResourceModelServingEndpoint) updateAiGateway(ctx context.Context, id s func (r *ResourceModelServingEndpoint) updateConfig(ctx context.Context, id string, config *serving.EndpointCoreConfigInput) error { if config == nil { // Unset config in resource. - req := serving.EndpointCoreConfigInput{ - Name: id, - AutoCaptureConfig: nil, - ServedEntities: nil, - TrafficConfig: nil, - ServedModels: nil, - } + var req serving.EndpointCoreConfigInput + req.Name = id _, err := r.client.ServingEndpoints.UpdateConfig(ctx, req) return err } - req := serving.EndpointCoreConfigInput{ - Name: id, - AutoCaptureConfig: config.AutoCaptureConfig, - ServedEntities: config.ServedEntities, - TrafficConfig: config.TrafficConfig, - ServedModels: config.ServedModels, - } + req := *config + req.Name = id _, err := r.client.ServingEndpoints.UpdateConfig(ctx, req) if err != nil { return fmt.Errorf("failed to update config: %w", err) diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 27b3d609357..40736048b6a 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -97,50 +96,16 @@ func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemo return makePipelineRemote(resp), nil } +// AllowDuplicateNames and DryRun are request-only, so PipelineSpec has no value for them. +var toCreatePipeline = newCopier[pipelines.PipelineSpec, pipelines.CreatePipeline]() + func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { var createPipeline pipelines.CreatePipeline if p.Spec != nil { - spec := p.Spec - createPipeline = pipelines.CreatePipeline{ - // Note: AllowDuplicateNames and DryRun are not in PipelineSpec, - // they are request-only fields, so they stay at their zero values. - AllowDuplicateNames: false, - BudgetPolicyId: spec.BudgetPolicyId, - Catalog: spec.Catalog, - Channel: spec.Channel, - Clusters: spec.Clusters, - Configuration: spec.Configuration, - Continuous: spec.Continuous, //nolint:staticcheck // SA1019: pipeline continuous is deprecated in the SDK but remains a supported bundle config field - Deployment: spec.Deployment, - Development: spec.Development, //nolint:staticcheck // SA1019: pipeline development is deprecated in the SDK but remains a supported bundle config field - DryRun: false, - Edition: spec.Edition, - Environment: spec.Environment, - EventLog: spec.EventLog, - Filters: spec.Filters, - GatewayDefinition: spec.GatewayDefinition, - Id: spec.Id, - IngestionDefinition: spec.IngestionDefinition, - Libraries: spec.Libraries, - Name: spec.Name, - Notifications: spec.Notifications, - Parameters: p.Parameters, - ParentPath: spec.ParentPath, - Photon: spec.Photon, - RestartWindow: spec.RestartWindow, - RootPath: spec.RootPath, - RunAs: p.RunAs, - Schema: spec.Schema, - Serverless: spec.Serverless, - ServerlessComputeId: spec.ServerlessComputeId, - Storage: spec.Storage, - Tags: spec.Tags, - Target: spec.Target, - Testing: spec.Testing, - Trigger: spec.Trigger, - UsagePolicyId: spec.UsagePolicyId, - ForceSendFields: utils.FilterFields[pipelines.CreatePipeline](spec.ForceSendFields, "AllowDuplicateNames", "DryRun", "Parameters", "RunAs"), - } + createPipeline = *toCreatePipeline.Copy(p.Spec) + // The response reports these next to the spec, not in it. + createPipeline.Parameters = p.Parameters + createPipeline.RunAs = p.RunAs } return &PipelineRemote{ CreatePipeline: createPipeline, @@ -167,6 +132,8 @@ func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) return response.PipelineId, nil, nil } +var toEditPipeline = newCopier[PipelineState, pipelines.EditPipeline]() + func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *PipelineState, entry *PlanEntry) (*PipelineRemote, error) { // cascade_on_destroy is a delete-time-only setting with no update API, so a change to it // alone must persist to state without a pipeline Update call. @@ -174,47 +141,10 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *Pipe return nil, nil } - request := pipelines.EditPipeline{ - AllowDuplicateNames: config.AllowDuplicateNames, - BudgetPolicyId: config.BudgetPolicyId, - Catalog: config.Catalog, - Channel: config.Channel, - Clusters: config.Clusters, - Configuration: config.Configuration, - Continuous: config.Continuous, //nolint:staticcheck // SA1019: pipeline continuous is deprecated in the SDK but remains a supported bundle config field - Deployment: config.Deployment, - Development: config.Development, //nolint:staticcheck // SA1019: pipeline development is deprecated in the SDK but remains a supported bundle config field - Edition: config.Edition, - Environment: config.Environment, - EventLog: config.EventLog, - ExpectedLastModified: 0, - Filters: config.Filters, - GatewayDefinition: config.GatewayDefinition, - Id: config.Id, - IngestionDefinition: config.IngestionDefinition, - Libraries: config.Libraries, - Name: config.Name, - Notifications: config.Notifications, - Parameters: config.Parameters, - ParentPath: config.ParentPath, - Photon: config.Photon, - RestartWindow: config.RestartWindow, - RootPath: config.RootPath, - RunAs: config.RunAs, - Schema: config.Schema, - Serverless: config.Serverless, - ServerlessComputeId: config.ServerlessComputeId, - Storage: config.Storage, - Tags: config.Tags, - Target: config.Target, - Testing: config.Testing, - Trigger: config.Trigger, - UsagePolicyId: config.UsagePolicyId, - PipelineId: id, - ForceSendFields: utils.FilterFields[pipelines.EditPipeline](config.ForceSendFields), - } + request := toEditPipeline.Copy(config) + request.PipelineId = id - return nil, r.client.Pipelines.Update(ctx, request) + return nil, r.client.Pipelines.Update(ctx, *request) } func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, state *PipelineState) error { diff --git a/bundle/direct/dresources/quality_monitor.go b/bundle/direct/dresources/quality_monitor.go index c66fed4e0bb..4b75332f95c 100644 --- a/bundle/direct/dresources/quality_monitor.go +++ b/bundle/direct/dresources/quality_monitor.go @@ -4,7 +4,7 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/utils" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -41,28 +41,18 @@ func (*ResourceQualityMonitor) PrepareState(input *resources.QualityMonitor) *Qu } } +var ( + toQualityMonitorState = newCopier[catalog.MonitorInfo, QualityMonitorState]() + + // TableName addresses the monitor, so it is the id. latest_monitor_failure_msg is the + // backend's latest error message, which the update API ignores. + toUpdateMonitor = newCopier[QualityMonitorState, catalog.UpdateMonitor]( + structcopy.Skip("TableName", "LatestMonitorFailureMsg"), + ) +) + func (*ResourceQualityMonitor) RemapState(info *catalog.MonitorInfo) *QualityMonitorState { - return &QualityMonitorState{ - CreateMonitor: catalog.CreateMonitor{ - AssetsDir: info.AssetsDir, - BaselineTableName: info.BaselineTableName, - CustomMetrics: info.CustomMetrics, - DataClassificationConfig: info.DataClassificationConfig, - InferenceLog: info.InferenceLog, - LatestMonitorFailureMsg: info.LatestMonitorFailureMsg, - Notifications: info.Notifications, - OutputSchemaName: info.OutputSchemaName, - Schedule: info.Schedule, - SkipBuiltinDashboard: false, - SlicingExprs: info.SlicingExprs, - Snapshot: info.Snapshot, - TableName: info.TableName, - TimeSeries: info.TimeSeries, - WarehouseId: "", - ForceSendFields: utils.FilterFields[catalog.CreateMonitor](info.ForceSendFields), - }, - TableName: info.TableName, - } + return toQualityMonitorState.Copy(info) } func (r *ResourceQualityMonitor) DoRead(ctx context.Context, id string) (*catalog.MonitorInfo, error) { @@ -84,25 +74,11 @@ func (r *ResourceQualityMonitor) DoCreate(ctx context.Context, config *QualityMo } func (r *ResourceQualityMonitor) DoUpdate(ctx context.Context, id string, config *QualityMonitorState, _ *PlanEntry) (*catalog.MonitorInfo, error) { - updateRequest := catalog.UpdateMonitor{ - TableName: id, - BaselineTableName: config.BaselineTableName, - CustomMetrics: config.CustomMetrics, - DashboardId: "", - DataClassificationConfig: config.DataClassificationConfig, - InferenceLog: config.InferenceLog, - LatestMonitorFailureMsg: "", - Notifications: config.Notifications, - OutputSchemaName: config.OutputSchemaName, - Schedule: config.Schedule, - SlicingExprs: config.SlicingExprs, - Snapshot: config.Snapshot, - TimeSeries: config.TimeSeries, - ForceSendFields: utils.FilterFields[catalog.UpdateMonitor](config.ForceSendFields), - } + updateRequest := toUpdateMonitor.Copy(config) + updateRequest.TableName = id //nolint:staticcheck // Direct quality_monitor resource still uses legacy monitor endpoints; v1 data-quality migration is separate work. - response, err := r.client.QualityMonitors.Update(ctx, updateRequest) + response, err := r.client.QualityMonitors.Update(ctx, *updateRequest) if err != nil { return nil, err } diff --git a/bundle/direct/dresources/registered_model.go b/bundle/direct/dresources/registered_model.go index 4352b848fe8..cb1e65a5769 100644 --- a/bundle/direct/dresources/registered_model.go +++ b/bundle/direct/dresources/registered_model.go @@ -3,12 +3,11 @@ package dresources import ( "context" "fmt" - "slices" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/catalog" ) @@ -30,28 +29,12 @@ func (*ResourceRegisteredModel) PrepareState(input *resources.RegisteredModel) * return &input.CreateRegisteredModelRequest } +// The output-only fields (created_at, updated_by, ...) are copied as well. Remote changes to +// them are ignored via ignore_remote_changes in registered_models.yml rather than zeroed here. +var toCreateRegisteredModel = newCopier[catalog.RegisteredModelInfo, catalog.CreateRegisteredModelRequest]() + func (*ResourceRegisteredModel) RemapState(model *catalog.RegisteredModelInfo) *catalog.CreateRegisteredModelRequest { - return &catalog.CreateRegisteredModelRequest{ - CatalogName: model.CatalogName, - Comment: model.Comment, - Name: model.Name, - SchemaName: model.SchemaName, - StorageLocation: model.StorageLocation, - ForceSendFields: utils.FilterFields[catalog.CreateRegisteredModelRequest](model.ForceSendFields), - - Aliases: model.Aliases, - BrowseOnly: model.BrowseOnly, - FullName: model.FullName, - MetastoreId: model.MetastoreId, - Owner: model.Owner, - - // Output only fields. Remote changes to these are ignored via - // ignore_remote_changes in registered_models.yml rather than zeroed here. - CreatedAt: model.CreatedAt, - CreatedBy: model.CreatedBy, - UpdatedAt: model.UpdatedAt, - UpdatedBy: model.UpdatedBy, - } + return toCreateRegisteredModel.Copy(model) } func (r *ResourceRegisteredModel) DoRead(ctx context.Context, id string) (*catalog.RegisteredModelInfo, error) { @@ -91,37 +74,25 @@ func (r *ResourceRegisteredModel) WaitAfterCreate(ctx context.Context, id string // See schemaForceSend. Verified against a real workspace: {"comment": ""} clears it. var registeredModelForceSend = []string{"Comment"} +// Not copied into an update: +// - Owner is settable in the config (it comes from the embedded CreateRegisteredModelRequest) +// and create sends it, but update never has: a change to it is silently dropped rather +// than applied. +// - Aliases are ignored by the update API; syncAliases applies them through the alias API. +// - FullName addresses the model, so it is the id. +// +// NewName stays empty: name updates are not supported yet without recreating. Can be added as +// a follow-up. TF does not support changing name without a recreate either. +var toUpdateRegisteredModel = newCopier[catalog.CreateRegisteredModelRequest, catalog.UpdateRegisteredModelRequest]( + structcopy.Skip("Owner", "Aliases", "FullName"), + structcopy.ForceSend(registeredModelForceSend...), +) + func (r *ResourceRegisteredModel) DoUpdate(ctx context.Context, id string, config *catalog.CreateRegisteredModelRequest, entry *PlanEntry) (*catalog.RegisteredModelInfo, error) { - updateRequest := catalog.UpdateRegisteredModelRequest{ - FullName: id, - Comment: config.Comment, - ForceSendFields: utils.FilterFields[catalog.UpdateRegisteredModelRequest](append(slices.Clone(registeredModelForceSend), config.ForceSendFields...), "Owner", "NewName"), - - // Owner is settable in the config (it comes from the embedded - // CreateRegisteredModelRequest) and create sends it, but update never has: a - // change to it is silently dropped rather than applied. - Owner: "", - - // Name updates are not supported yet without recreating. Can be added as a follow-up. - // Note: TF also does not support changing name without a recreate so the current behavior matches TF. - NewName: "", - - // The update API ignores aliases; syncAliases applies them through the alias API. - Aliases: nil, - - BrowseOnly: config.BrowseOnly, - CreatedAt: config.CreatedAt, - CreatedBy: config.CreatedBy, - MetastoreId: config.MetastoreId, - UpdatedAt: config.UpdatedAt, - UpdatedBy: config.UpdatedBy, - SchemaName: config.SchemaName, - StorageLocation: config.StorageLocation, - Name: config.Name, - CatalogName: config.CatalogName, - } + updateRequest := toUpdateRegisteredModel.Copy(config) + updateRequest.FullName = id - response, err := r.client.RegisteredModels.Update(ctx, updateRequest) + response, err := r.client.RegisteredModels.Update(ctx, *updateRequest) if err != nil { return nil, err } diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 76a619ee8ad..7e3c587a88b 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -2,11 +2,10 @@ package dresources import ( "context" - "slices" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/utils" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -43,17 +42,10 @@ func (*ResourceSchema) PrepareState(input *resources.Schema) *SchemaState { } } +var toSchemaState = newCopier[catalog.SchemaInfo, SchemaState]() + func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *SchemaState { - return &SchemaState{ - CatalogName: info.CatalogName, - Comment: info.Comment, - CustomMaxRetentionHours: info.CustomMaxRetentionHours, - Name: info.Name, - Properties: info.Properties, - StorageRoot: info.StorageRoot, - ForceSendFields: utils.FilterFields[catalog.CreateSchema](info.ForceSendFields), - SchemaId: info.SchemaId, - } + return toSchemaState.Copy(info) } func (r *ResourceSchema) DoRead(ctx context.Context, id string) (*catalog.SchemaInfo, error) { @@ -85,17 +77,13 @@ func (r *ResourceSchema) DoCreate(ctx context.Context, config *SchemaState) (str // migrate invariant (schema.yml is excluded there, and schema_tf_compat.yml covers migrate instead). var schemaForceSend = []string{"Comment", "CustomMaxRetentionHours"} +var toUpdateSchema = newCopier[catalog.CreateSchema, catalog.UpdateSchema](structcopy.ForceSend(schemaForceSend...)) + func makeUpdateSchema(id, newName string, config *catalog.CreateSchema) catalog.UpdateSchema { - return catalog.UpdateSchema{ - Comment: config.Comment, - CustomMaxRetentionHours: config.CustomMaxRetentionHours, - EnablePredictiveOptimization: "", // Not supported by DABs - FullName: id, - NewName: newName, - Owner: "", // Not supported by DABs - Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateSchema](append(slices.Clone(schemaForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "NewName", "Owner"), - } + request := toUpdateSchema.Copy(config) + request.FullName = id + request.NewName = newName + return *request } // DoUpdate updates the schema in place and returns remote state. diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 7cc30eec4d5..28c83d5a68f 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -2,13 +2,12 @@ package dresources import ( "context" - "slices" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/common/types/fieldmask" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -41,7 +40,7 @@ func (*ResourceSecret) PrepareState(input *resources.Secret) *catalog.Secret { MetastoreId: "", UpdateTime: nil, UpdatedBy: "", - ForceSendFields: utils.FilterFields[catalog.Secret](nil), + ForceSendFields: nil, } } @@ -87,13 +86,13 @@ func (r *ResourceSecret) DoCreate(ctx context.Context, state *catalog.Secret) (s // body and the old value would drift back. Verified against a real workspace: {"comment": ""} clears it. var secretForceSend = []string{"Comment"} +var toUpdateSecret = newCopier[catalog.Secret, catalog.Secret](structcopy.ForceSend(secretForceSend...)) + // DoUpdate updates the secret in place and returns remote state. func (r *ResourceSecret) DoUpdate(ctx context.Context, id string, state *catalog.Secret, _ *PlanEntry) (*catalog.Secret, error) { - secret := *state - secret.ForceSendFields = utils.FilterFields[catalog.Secret](append(slices.Clone(secretForceSend), state.ForceSendFields...)) response, err := r.client.SecretsUc.UpdateSecret(ctx, catalog.UpdateSecretRequest{ FullName: id, - Secret: secret, + Secret: *toUpdateSecret.Copy(state), UpdateMask: fieldmask.FieldMask{ Paths: []string{"*"}, }, diff --git a/bundle/direct/dresources/secret_scope.go b/bundle/direct/dresources/secret_scope.go index 7ad0b0d469a..4e9936025ca 100644 --- a/bundle/direct/dresources/secret_scope.go +++ b/bundle/direct/dresources/secret_scope.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" @@ -59,14 +58,10 @@ func (*ResourceSecretScope) PrepareState(input *resources.SecretScope) *SecretSc } } +var toSecretScopeConfig = newCopier[SecretScopeRemote, SecretScopeConfig]() + func (*ResourceSecretScope) RemapState(remote *SecretScopeRemote) *SecretScopeConfig { - return &SecretScopeConfig{ - Scope: remote.Scope, - ScopeBackendType: remote.ScopeBackendType, - BackendAzureKeyvault: remote.BackendAzureKeyvault, - InitialManagePrincipal: "", - ForceSendFields: utils.FilterFields[workspace.CreateScope](remote.ForceSendFields), - } + return toSecretScopeConfig.Copy(remote) } // DoRead fetches the secret scope by name. Since the Secrets API does not provide diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index f7b9ca09fbe..145f6f4fa18 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/sql" @@ -68,30 +67,15 @@ func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *SqlWar return s } +var ( + toSqlWarehouseState = newCopier[SqlWarehouseRemote, SqlWarehouseState]() + toEditWarehouse = newCopier[SqlWarehouseState, sql.EditWarehouseRequest]() +) + // RemapState maps the remote SqlWarehouseRemote to SqlWarehouseState for diff comparison. // Started is derived from warehouse state so the planner can detect start/stop changes. func (*ResourceSqlWarehouse) RemapState(warehouse *SqlWarehouseRemote) *SqlWarehouseState { - return &SqlWarehouseState{ - AutoStopMins: warehouse.AutoStopMins, - Channel: warehouse.Channel, - ClusterSize: warehouse.ClusterSize, - CreatorName: warehouse.CreatorName, - CustomerVersionLabel: warehouse.CustomerVersionLabel, - EnablePhoton: warehouse.EnablePhoton, - EnableServerlessCompute: warehouse.EnableServerlessCompute, - InstanceProfileArn: warehouse.InstanceProfileArn, - MaxDbu: warehouse.MaxDbu, - MaxNumClusters: warehouse.MaxNumClusters, - MinNumClusters: warehouse.MinNumClusters, - Name: warehouse.Name, - ReydenClusterSizeMultiplier: warehouse.ReydenClusterSizeMultiplier, - SpotInstancePolicy: warehouse.SpotInstancePolicy, - StatementTimeout: warehouse.StatementTimeout, - Tags: warehouse.Tags, - WarehouseType: sql.CreateWarehouseRequestWarehouseType(warehouse.WarehouseType), - ForceSendFields: utils.FilterFields[sql.CreateWarehouseRequest](warehouse.ForceSendFields), - Lifecycle: warehouse.Lifecycle, - } + return toSqlWarehouseState.Copy(warehouse) } // DoRead reads the warehouse by id. @@ -137,29 +121,10 @@ func hasWarehouseChanges(entry *PlanEntry) bool { func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *SqlWarehouseState, entry *PlanEntry) (*SqlWarehouseRemote, error) { edited := hasWarehouseChanges(entry) if edited { - request := sql.EditWarehouseRequest{ - AutoStopMins: config.AutoStopMins, - Channel: config.Channel, - ClusterSize: config.ClusterSize, - CreatorName: config.CreatorName, - CustomerVersionLabel: config.CustomerVersionLabel, - EnablePhoton: config.EnablePhoton, - EnableServerlessCompute: config.EnableServerlessCompute, - Id: id, - InstanceProfileArn: config.InstanceProfileArn, - MaxDbu: config.MaxDbu, - MaxNumClusters: config.MaxNumClusters, - MinNumClusters: config.MinNumClusters, - Name: config.Name, - ReydenClusterSizeMultiplier: config.ReydenClusterSizeMultiplier, - SpotInstancePolicy: config.SpotInstancePolicy, - StatementTimeout: config.StatementTimeout, - Tags: config.Tags, - WarehouseType: sql.EditWarehouseRequestWarehouseType(config.WarehouseType), - ForceSendFields: utils.FilterFields[sql.EditWarehouseRequest](config.ForceSendFields), - } + request := toEditWarehouse.Copy(config) + request.Id = id - waiter, err := r.client.Warehouses.Edit(ctx, request) + waiter, err := r.client.Warehouses.Edit(ctx, *request) if err != nil { return nil, err } diff --git a/bundle/direct/dresources/testdata/field_copies.txt b/bundle/direct/dresources/testdata/field_copies.txt new file mode 100644 index 00000000000..3de36791e59 --- /dev/null +++ b/bundle/direct/dresources/testdata/field_copies.txt @@ -0,0 +1,626 @@ +catalog.CatalogInfo -> dresources.CatalogState + Comment: copied + ConnectionName: copied + CustomMaxRetentionHours: copied + IsolationMode: copied + ManagedEncryptionSettings: copied + Name: copied + Options: copied + Properties: copied + ProviderName: copied + ShareName: copied + StorageRoot: copied + +catalog.CreateExternalLocation -> catalog.UpdateExternalLocation + Comment: copied, always sent + CredentialName: copied + EffectiveEnableFileEvents: skipped + EffectiveFileEventQueue: skipped + EnableFileEvents: copied + EncryptionDetails: copied + Fallback: copied + FileEventQueue: copied + Force: not in source + IsolationMode: not in source + Name: skipped + NewName: not in source + Owner: not in source + ReadOnly: copied + SkipValidation: copied + Url: copied + +catalog.CreateRegisteredModelRequest -> catalog.UpdateRegisteredModelRequest + Aliases: skipped + BrowseOnly: copied + CatalogName: copied + Comment: copied, always sent + CreatedAt: copied + CreatedBy: copied + FullName: skipped + MetastoreId: copied + Name: copied + NewName: not in source + Owner: skipped + SchemaName: copied + StorageLocation: copied + UpdatedAt: copied + UpdatedBy: copied + +catalog.CreateSchema -> catalog.UpdateSchema + Comment: copied, always sent + CustomMaxRetentionHours: copied, always sent + EnablePredictiveOptimization: not in source + FullName: not in source + NewName: not in source + Owner: not in source + Properties: copied + +catalog.ExternalLocationInfo -> catalog.CreateExternalLocation + Comment: copied + CredentialName: copied + EffectiveEnableFileEvents: copied + EffectiveFileEventQueue: copied + EnableFileEvents: copied + EncryptionDetails: copied + Fallback: copied + FileEventQueue: copied + Name: copied + ReadOnly: copied + SkipValidation: not in source + Url: copied + +catalog.MonitorInfo -> dresources.QualityMonitorState + AssetsDir: copied + BaselineTableName: copied + CustomMetrics: copied + DataClassificationConfig: copied + InferenceLog: copied + LatestMonitorFailureMsg: copied + Notifications: copied + OutputSchemaName: copied + Schedule: copied + SkipBuiltinDashboard: not in source + SlicingExprs: copied + Snapshot: copied + TableName: copied + TimeSeries: copied + WarehouseId: not in source + +catalog.RegisteredModelInfo -> catalog.CreateRegisteredModelRequest + Aliases: copied + BrowseOnly: copied + CatalogName: copied + Comment: copied + CreatedAt: copied + CreatedBy: copied + FullName: copied + MetastoreId: copied + Name: copied + Owner: copied + SchemaName: copied + StorageLocation: copied + UpdatedAt: copied + UpdatedBy: copied + +catalog.SchemaInfo -> dresources.SchemaState + CatalogName: copied + Comment: copied + CustomMaxRetentionHours: copied + Name: copied + Properties: copied + SchemaId: copied + StorageRoot: copied + +catalog.Secret -> catalog.Secret + CatalogName: copied + Comment: copied, always sent + CreateTime: copied + CreatedBy: copied + EffectiveOwner: copied + EffectiveValue: copied + ExpireTime: copied + FullName: copied + MetastoreId: copied + Name: copied + Owner: copied + SchemaName: copied + UpdateTime: copied + UpdatedBy: copied + Value: copied + +catalog.VolumeInfo -> dresources.VolumeState + CatalogName: copied + Comment: copied + Name: copied + SchemaName: copied + StorageLocation: copied + VolumeId: copied + VolumeType: copied + +compute.ClusterSpec -> compute.CreateCluster + ApplyPolicyDefaultValues: copied + Autoscale: copied + AutoterminationMinutes: copied + AwsAttributes: copied + AzureAttributes: copied + CloneFrom: not in source + ClusterLogConf: copied + ClusterName: copied + CustomTags: copied + DataSecurityMode: copied + DependencyMode: copied + DockerImage: copied + DriverInstancePoolId: copied + DriverNodeTypeFlexibility: copied + DriverNodeTypeId: copied + EnableElasticDisk: copied + EnableLocalDiskEncryption: copied + GcpAttributes: copied + InitScripts: copied + InstancePoolId: copied + IsSingleNode: copied + Kind: copied + NodeTypeId: copied + NumWorkers: copied + PolicyId: copied + RemoteDiskThroughput: copied + RuntimeEngine: copied + SingleUserName: copied + SparkConf: copied + SparkEnvVars: copied + SparkVersion: copied + SshPublicKeys: copied + TotalInitialRemoteDiskSize: copied + UseMlRuntime: copied + WorkerNodeTypeFlexibility: copied + WorkloadType: copied + +compute.ClusterSpec -> compute.EditCluster + ApplyPolicyDefaultValues: copied + Autoscale: copied + AutoterminationMinutes: copied + AwsAttributes: copied + AzureAttributes: copied + ClusterId: not in source + ClusterLogConf: copied + ClusterName: copied + CustomTags: copied + DataSecurityMode: copied + DependencyMode: copied + DockerImage: copied + DriverInstancePoolId: copied + DriverNodeTypeFlexibility: copied + DriverNodeTypeId: copied + EnableElasticDisk: copied + EnableLocalDiskEncryption: copied + GcpAttributes: copied + InitScripts: copied + InstancePoolId: copied + IsSingleNode: copied + Kind: copied + NodeTypeId: copied + NumWorkers: copied + PolicyId: copied + RemoteDiskThroughput: copied + RuntimeEngine: copied + SingleUserName: copied + SparkConf: copied + SparkEnvVars: copied + SparkVersion: copied + SshPublicKeys: copied + TotalInitialRemoteDiskSize: copied + UseMlRuntime: copied + WorkerNodeTypeFlexibility: copied + WorkloadType: copied + +compute.CreateInstancePool -> compute.EditInstancePool + CustomTags: copied + IdleInstanceAutoterminationMinutes: copied + InstancePoolId: not in source + InstancePoolName: copied + MaxCapacity: copied + MinIdleInstances: copied + NodeTypeId: copied + RemoteDiskThroughput: copied + TotalInitialRemoteDiskSize: copied + +compute.CreatePolicy -> compute.EditPolicy + Definition: copied + Description: copied + Libraries: copied + MaxClustersPerUser: copied + Name: copied + PolicyFamilyDefinitionOverrides: copied + PolicyFamilyId: copied + PolicyId: not in source + +compute.GetInstancePool -> compute.CreateInstancePool + AwsAttributes: copied + AzureAttributes: copied + CustomTags: copied + DiskSpec: copied + EnableElasticDisk: copied + GcpAttributes: copied + IdleInstanceAutoterminationMinutes: copied + InstancePoolName: copied + MaxCapacity: copied + MinIdleInstances: copied + NodeTypeFlexibility: copied + NodeTypeId: copied + PreloadedDockerImages: copied + PreloadedSparkVersions: copied + RemoteDiskThroughput: copied + TotalInitialRemoteDiskSize: copied + +compute.Policy -> compute.CreatePolicy + Definition: copied + Description: copied + Libraries: copied + MaxClustersPerUser: copied + Name: copied + PolicyFamilyDefinitionOverrides: copied + PolicyFamilyId: copied + +dresources.CatalogState -> catalog.UpdateCatalog + Comment: copied, always sent + CustomMaxRetentionHours: copied, always sent + EnablePredictiveOptimization: not in source + IsolationMode: copied + ManagedEncryptionSettings: copied + Name: skipped + NewName: not in source + Options: copied + Owner: not in source + Properties: copied + +dresources.ClusterRemote -> dresources.ClusterState + ApplyPolicyDefaultValues: copied + Autoscale: copied + AutoterminationMinutes: copied + AwsAttributes: copied + AzureAttributes: copied + ClusterLogConf: copied + ClusterName: copied + CustomTags: copied + DataSecurityMode: copied + DependencyMode: copied + DockerImage: copied + DriverInstancePoolId: copied + DriverNodeTypeFlexibility: copied + DriverNodeTypeId: copied + EnableElasticDisk: copied + EnableLocalDiskEncryption: copied + GcpAttributes: copied + InitScripts: copied + InstancePoolId: copied + IsSingleNode: copied + Kind: copied + Libraries: copied + Lifecycle: copied + NodeTypeId: copied + NumWorkers: copied + PolicyId: copied + RemoteDiskThroughput: copied + RuntimeEngine: copied + SingleUserName: copied + SparkConf: copied + SparkEnvVars: copied + SparkVersion: copied + SshPublicKeys: copied + TotalInitialRemoteDiskSize: copied + UseMlRuntime: copied + WorkerNodeTypeFlexibility: copied + WorkloadType: copied + +dresources.ClusterState -> compute.ResizeCluster + Autoscale: copied + ClusterId: not in source + NumWorkers: copied + +dresources.MlflowModelRemote -> ml.CreateModelRequest + Description: copied + Name: copied + Tags: copied + +dresources.ModelServingEndpointRemote -> serving.CreateServingEndpoint + AiGateway: copied + BudgetPolicyId: copied + Config: copied + Description: copied + EmailNotifications: copied + Name: copied + RateLimits: not in source + RouteOptimized: copied + Tags: copied + TelemetryConfig: copied + +dresources.PipelineState -> pipelines.EditPipeline + AllowDuplicateNames: copied + BudgetPolicyId: copied + Catalog: copied + Channel: copied + Clusters: copied + Configuration: copied + Continuous: copied + Deployment: copied + Development: copied + Edition: copied + Environment: copied + EventLog: copied + ExpectedLastModified: not in source + Filters: copied + GatewayDefinition: copied + Id: copied + IngestionDefinition: copied + Libraries: copied + Name: copied + Notifications: copied + Parameters: copied + ParentPath: copied + Photon: copied + PipelineId: not in source + RestartWindow: copied + RootPath: copied + RunAs: copied + Schema: copied + Serverless: copied + ServerlessComputeId: copied + Storage: copied + Tags: copied + Target: copied + Testing: copied + Trigger: copied + UsagePolicyId: copied + +dresources.QualityMonitorState -> catalog.UpdateMonitor + BaselineTableName: copied + CustomMetrics: copied + DashboardId: not in source + DataClassificationConfig: copied + InferenceLog: copied + LatestMonitorFailureMsg: skipped + Notifications: copied + OutputSchemaName: copied + Schedule: copied + SlicingExprs: copied + Snapshot: copied + TableName: skipped + TimeSeries: copied + +dresources.SecretScopeRemote -> dresources.SecretScopeConfig + BackendAzureKeyvault: copied + InitialManagePrincipal: not in source + Scope: copied + ScopeBackendType: copied + +dresources.SqlWarehouseRemote -> dresources.SqlWarehouseState + AutoStopMins: copied + Channel: copied + ClusterSize: copied + CreatorName: copied + CustomerVersionLabel: copied + EnablePhoton: copied + EnableServerlessCompute: copied + InstanceProfileArn: copied + Lifecycle: copied + MaxDbu: copied + MaxNumClusters: copied + MinNumClusters: copied + Name: copied + ReydenClusterSizeMultiplier: copied + SpotInstancePolicy: copied + StatementTimeout: copied + Tags: copied + WarehouseType: copied, converted from sql.GetWarehouseResponseWarehouseType + +dresources.SqlWarehouseState -> sql.EditWarehouseRequest + AutoStopMins: copied + Channel: copied + ClusterSize: copied + CreatorName: copied + CustomerVersionLabel: copied + EnablePhoton: copied + EnableServerlessCompute: copied + Id: not in source + InstanceProfileArn: copied + MaxDbu: copied + MaxNumClusters: copied + MinNumClusters: copied + Name: copied + ReydenClusterSizeMultiplier: copied + SpotInstancePolicy: copied + StatementTimeout: copied + Tags: copied + WarehouseType: copied, converted from sql.CreateWarehouseRequestWarehouseType + +dresources.VectorSearchEndpointRemote -> vectorsearch.CreateEndpoint + BudgetPolicyId: copied + EndpointType: copied + Name: copied + TargetQps: copied + UsagePolicyId: not in source + +dresources.VectorSearchIndexRemote -> dresources.VectorSearchIndexState + DeltaSyncIndexSpec: skipped + DirectAccessIndexSpec: copied + EndpointName: copied + EndpointUuid: copied + IndexSubtype: copied + IndexType: copied + Name: copied + PrimaryKey: copied + +dresources.VolumeState -> catalog.UpdateVolumeRequestContent + Comment: copied, always sent + Name: skipped + NewName: not in source + Owner: not in source + +jobs.JobSettings -> jobs.CreateJob + AccessControlList: not in source + BudgetPolicyId: copied + Continuous: copied + Deployment: copied + Description: copied + DurationTarget: copied + EditMode: copied + EmailNotifications: copied + EnvironmentVariables: copied + Environments: copied + Format: copied + GitSource: copied + Health: copied + JobClusters: copied + MaxConcurrentRuns: copied + Name: copied + NotificationSettings: copied + Parameters: copied + ParentPath: copied + PerformanceTarget: copied + Queue: copied + RunAs: copied + Schedule: copied + Tags: copied + Tasks: copied + TimeoutSeconds: copied + Trigger: copied + Triggers: copied + UsagePolicyId: copied + WebhookNotifications: copied + +ml.CreateModelRequest -> ml.UpdateModelRequest + Description: copied + Name: skipped + +ml.Experiment -> ml.CreateExperiment + ArtifactLocation: copied + Name: copied + Tags: copied + TraceLocation: copied + +ml.Feature -> ml.Feature + CatalogName: copied + CreatedAt: copied + CreatedBy: copied + Description: copied, always sent + Entities: copied + FilterCondition: copied + FullName: copied + Function: copied + Inputs: copied + LineageContext: copied + Name: copied + OutputTypeText: copied + SchemaName: copied + Source: copied + TimeWindow: copied + TimeseriesColumn: copied + +ml.Model -> dresources.MlflowModelRemote + CreationTimestamp: copied + Description: copied + Id: not in source + LastUpdatedTimestamp: copied + LatestVersions: copied + ModelId: not in source + Name: copied + PermissionLevel: not in source + Tags: copied + UserId: copied + +pipelines.PipelineSpec -> pipelines.CreatePipeline + AllowDuplicateNames: not in source + BudgetPolicyId: copied + Catalog: copied + Channel: copied + Clusters: copied + Configuration: copied + Continuous: copied + Deployment: copied + Development: copied + DryRun: not in source + Edition: copied + Environment: copied + EventLog: copied + Filters: copied + GatewayDefinition: copied + Id: copied + IngestionDefinition: copied + Libraries: copied + Name: copied + Notifications: copied + Parameters: not in source + ParentPath: copied + Photon: copied + RestartWindow: copied + RootPath: copied + RunAs: not in source + Schema: copied + Serverless: copied + ServerlessComputeId: copied + Storage: copied + Tags: copied + Target: copied + Testing: copied + Trigger: copied + UsagePolicyId: copied + +serving.AiGatewayConfig -> serving.PutAiGatewayRequest + FallbackConfig: copied + Guardrails: copied + InferenceTableConfig: copied + Name: not in source + RateLimits: copied + UsageTrackingConfig: copied + +serving.AutoCaptureConfigOutput -> serving.AutoCaptureConfigInput + CatalogName: copied + Enabled: copied + SchemaName: copied + TableNamePrefix: copied + +serving.EndpointCoreConfigOutput -> serving.EndpointCoreConfigInput + AutoCaptureConfig: skipped + Name: not in source + ServedEntities: skipped + ServedModels: skipped + TrafficConfig: copied + +serving.ServedEntityOutput -> serving.ServedEntityInput + BurstScalingEnabled: copied + EntityName: copied + EntityVersion: copied + EnvironmentVars: copied + ExternalModel: copied + InstanceProfileArn: copied + MaxProvisionedConcurrency: copied + MaxProvisionedThroughput: copied + MinProvisionedConcurrency: copied + MinProvisionedThroughput: copied + Name: copied + ProvisionedModelUnits: copied + ScaleToZeroEnabled: copied + WorkloadSize: copied + WorkloadType: copied + +serving.ServingEndpointDetailed -> dresources.ModelServingEndpointRemote + AiGateway: copied + BudgetPolicyId: copied + Config: skipped + Description: copied + EmailNotifications: copied + EndpointDetails: not in source + EndpointId: not in source + Name: copied + RouteOptimized: copied + Tags: copied + TelemetryConfig: copied + +vectorsearch.DeltaSyncVectorIndexSpecResponse -> vectorsearch.DeltaSyncVectorIndexSpecRequest + ColumnsToIndex: copied + ColumnsToSync: copied + EmbeddingSourceColumns: copied + EmbeddingVectorColumns: copied + EmbeddingWritebackTable: copied + PipelineType: copied + SourceTable: copied diff --git a/bundle/direct/dresources/vector_search_endpoint.go b/bundle/direct/dresources/vector_search_endpoint.go index 4e39815fceb..f1b45ac1198 100644 --- a/bundle/direct/dresources/vector_search_endpoint.go +++ b/bundle/direct/dresources/vector_search_endpoint.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/vectorsearch" @@ -63,15 +62,10 @@ func (*ResourceVectorSearchEndpoint) PrepareState(input *resources.VectorSearchE return &input.CreateEndpoint } +var toCreateVectorSearchEndpoint = newCopier[VectorSearchEndpointRemote, vectorsearch.CreateEndpoint]() + func (*ResourceVectorSearchEndpoint) RemapState(remote *VectorSearchEndpointRemote) *vectorsearch.CreateEndpoint { - return &vectorsearch.CreateEndpoint{ - Name: remote.Name, - EndpointType: remote.EndpointType, - BudgetPolicyId: remote.BudgetPolicyId, - UsagePolicyId: "", // Missing in remote - TargetQps: remote.TargetQps, - ForceSendFields: utils.FilterFields[vectorsearch.CreateEndpoint](remote.ForceSendFields, "UsagePolicyId"), - } + return toCreateVectorSearchEndpoint.Copy(remote) } func (r *ResourceVectorSearchEndpoint) DoRead(ctx context.Context, id string) (*VectorSearchEndpointRemote, error) { diff --git a/bundle/direct/dresources/vector_search_index.go b/bundle/direct/dresources/vector_search_index.go index 857e3fc44db..981427e7c25 100644 --- a/bundle/direct/dresources/vector_search_index.go +++ b/bundle/direct/dresources/vector_search_index.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -81,32 +82,23 @@ func (*ResourceVectorSearchIndex) PrepareState(input *resources.VectorSearchInde } } +var ( + // DeltaSyncIndexSpec has a type of its own in the response, so it is converted on its own. + toVectorSearchIndexState = newCopier[VectorSearchIndexRemote, VectorSearchIndexState]( + structcopy.Skip("DeltaSyncIndexSpec"), + ) + toDeltaSyncIndexSpecRequest = newCopier[vectorsearch.DeltaSyncVectorIndexSpecResponse, vectorsearch.DeltaSyncVectorIndexSpecRequest]() +) + func (*ResourceVectorSearchIndex) RemapState(remote *VectorSearchIndexRemote) *VectorSearchIndexState { - state := &VectorSearchIndexState{ - DeltaSyncIndexSpec: nil, // need to remap below - DirectAccessIndexSpec: remote.DirectAccessIndexSpec, - IndexSubtype: remote.IndexSubtype, - Name: remote.Name, - EndpointName: remote.EndpointName, - IndexType: remote.IndexType, - PrimaryKey: remote.PrimaryKey, - EndpointUuid: remote.EndpointUuid, - } + state := toVectorSearchIndexState.Copy(remote) if remote.DeltaSyncIndexSpec != nil { - state.DeltaSyncIndexSpec = &vectorsearch.DeltaSyncVectorIndexSpecRequest{ - ColumnsToIndex: remote.DeltaSyncIndexSpec.ColumnsToIndex, - ColumnsToSync: remote.DeltaSyncIndexSpec.ColumnsToSync, - EmbeddingSourceColumns: remote.DeltaSyncIndexSpec.EmbeddingSourceColumns, - EmbeddingVectorColumns: remote.DeltaSyncIndexSpec.EmbeddingVectorColumns, - EmbeddingWritebackTable: remote.DeltaSyncIndexSpec.EmbeddingWritebackTable, - PipelineType: remote.DeltaSyncIndexSpec.PipelineType, - SourceTable: remote.DeltaSyncIndexSpec.SourceTable, - // ForceSendFields is an SDK marshaling concern (which zero-valued - // fields to wire-serialize) that has no meaning on the read path. - // Local config doesn't carry one either, so leave it nil rather - // than copy whatever the response struct happened to use. - ForceSendFields: nil, - } + state.DeltaSyncIndexSpec = toDeltaSyncIndexSpecRequest.Copy(remote.DeltaSyncIndexSpec) + // ForceSendFields is an SDK marshaling concern (which zero-valued + // fields to wire-serialize) that has no meaning on the read path. + // Local config doesn't carry one either, so leave it nil rather + // than copy whatever the response struct happened to use. + state.DeltaSyncIndexSpec.ForceSendFields = nil } return state } diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 63a978c01c2..5d819520fc9 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -3,12 +3,11 @@ package dresources import ( "context" "fmt" - "slices" "strings" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/utils" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -45,17 +44,10 @@ func (*ResourceVolume) PrepareState(input *resources.Volume) *VolumeState { } } +var toVolumeState = newCopier[catalog.VolumeInfo, VolumeState]() + func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *VolumeState { - return &VolumeState{ - CatalogName: info.CatalogName, - Comment: info.Comment, - Name: info.Name, - SchemaName: info.SchemaName, - StorageLocation: info.StorageLocation, - VolumeType: info.VolumeType, - ForceSendFields: utils.FilterFields[catalog.CreateVolumeRequestContent](info.ForceSendFields), - VolumeId: info.VolumeId, - } + return toVolumeState.Copy(info) } func (r *ResourceVolume) DoRead(ctx context.Context, id string) (*catalog.VolumeInfo, error) { @@ -73,15 +65,16 @@ func (r *ResourceVolume) DoCreate(ctx context.Context, config *VolumeState) (str // See schemaForceSend. Verified against a real workspace: {"comment": ""} clears it. var volumeForceSend = []string{"Comment"} -func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *VolumeState, _ *PlanEntry) (*catalog.VolumeInfo, error) { - updateRequest := catalog.UpdateVolumeRequestContent{ - Comment: config.Comment, - Name: id, - NewName: "", // Not supported by Update(). Needs DoUpdateWithID() - Owner: "", // Not supported by DABs +// Name addresses the volume by its full name; the name in the config goes in NewName when it +// differs, which only DoUpdateWithID does. +var toUpdateVolume = newCopier[VolumeState, catalog.UpdateVolumeRequestContent]( + structcopy.Skip("Name"), + structcopy.ForceSend(volumeForceSend...), +) - ForceSendFields: utils.FilterFields[catalog.UpdateVolumeRequestContent](append(slices.Clone(volumeForceSend), config.ForceSendFields...), "NewName", "Owner"), - } +func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *VolumeState, _ *PlanEntry) (*catalog.VolumeInfo, error) { + updateRequest := toUpdateVolume.Copy(config) + updateRequest.Name = id nameFromID, err := getNameFromID(id) if err != nil { @@ -92,7 +85,7 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *Volume return nil, fmt.Errorf("internal error: unexpected change of name from %#v to %#v", nameFromID, config.Name) } - response, err := r.client.Volumes.Update(ctx, updateRequest) + response, err := r.client.Volumes.Update(ctx, *updateRequest) if err != nil { return nil, err } @@ -105,15 +98,7 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *Volume } func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config *VolumeState, _ *PlanEntry) (string, *catalog.VolumeInfo, error) { - updateRequest := catalog.UpdateVolumeRequestContent{ - Comment: config.Comment, - Name: "", // Initialized below - - NewName: "", // Initialized below if needed - Owner: "", // Not supported by DABs - - ForceSendFields: utils.FilterFields[catalog.UpdateVolumeRequestContent](append(slices.Clone(volumeForceSend), config.ForceSendFields...), "Owner"), - } + updateRequest := toUpdateVolume.Copy(config) items := strings.Split(id, ".") if len(items) == 0 { @@ -129,7 +114,7 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string, config * updateRequest.NewName = config.Name } - response, err := r.client.Volumes.Update(ctx, updateRequest) + response, err := r.client.Volumes.Update(ctx, *updateRequest) if err != nil || response == nil { return "", nil, err } From 0b79d6c8317eb811ec41cbdc4bb6327b56f1cc3c Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 19 Nov 2026 01:13:12 +0000 Subject: [PATCH 3/9] direct: keep the notes on fields a request leaves unset The struct literals that the copiers replaced said why a field stayed zero (not supported by DABs, input-only, missing in remote). Carry those notes over to the copier declarations. Co-authored-by: Isaac --- bundle/direct/dresources/catalog.go | 1 + bundle/direct/dresources/cluster.go | 1 + bundle/direct/dresources/external_location.go | 2 ++ bundle/direct/dresources/schema.go | 1 + bundle/direct/dresources/vector_search_endpoint.go | 1 + bundle/direct/dresources/volume.go | 2 +- 6 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index a34860da80a..b7f66acf32c 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -86,6 +86,7 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s var catalogForceSend = []string{"Comment", "CustomMaxRetentionHours"} // Name addresses the catalog, so it is the id; the name in the config goes in NewName when it differs. +// EnablePredictiveOptimization and Owner are not supported by DABs. var toUpdateCatalog = newCopier[CatalogState, catalog.UpdateCatalog]( structcopy.Skip("Name"), structcopy.ForceSend(catalogForceSend...), diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 86644741ac3..d54d70f3b31 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -386,6 +386,7 @@ func remoteClusterIsRunning(entry *PlanEntry) bool { } var ( + // CloneFrom is not supported by DABs. toCreateCluster = newCopier[compute.ClusterSpec, compute.CreateCluster]() toEditCluster = newCopier[compute.ClusterSpec, compute.EditCluster]() toResizeCluster = newCopier[ClusterState, compute.ResizeCluster]() diff --git a/bundle/direct/dresources/external_location.go b/bundle/direct/dresources/external_location.go index c3b00645959..58a3f9c8759 100644 --- a/bundle/direct/dresources/external_location.go +++ b/bundle/direct/dresources/external_location.go @@ -22,6 +22,7 @@ func (*ResourceExternalLocation) PrepareState(input *resources.ExternalLocation) } // The output-only effective_* fields are mirrored into state to avoid churn in remapped config. +// SkipValidation is an input-only parameter, never returned by the API. var toCreateExternalLocation = newCopier[catalog.ExternalLocationInfo, catalog.CreateExternalLocation]() func (*ResourceExternalLocation) RemapState(info *catalog.ExternalLocationInfo) *catalog.CreateExternalLocation { @@ -49,6 +50,7 @@ var externalLocationForceSend = []string{"Comment"} // The effective_* fields are output-only, so they are never sent in an update. Name addresses // the location, so it is the id; the name in the config goes in NewName when it differs. +// IsolationMode and Owner are not supported by DABs. var toUpdateExternalLocation = newCopier[catalog.CreateExternalLocation, catalog.UpdateExternalLocation]( structcopy.Skip("EffectiveEnableFileEvents", "EffectiveFileEventQueue", "Name"), structcopy.ForceSend(externalLocationForceSend...), diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 7e3c587a88b..2bb296be75d 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -77,6 +77,7 @@ func (r *ResourceSchema) DoCreate(ctx context.Context, config *SchemaState) (str // migrate invariant (schema.yml is excluded there, and schema_tf_compat.yml covers migrate instead). var schemaForceSend = []string{"Comment", "CustomMaxRetentionHours"} +// EnablePredictiveOptimization and Owner are not supported by DABs. var toUpdateSchema = newCopier[catalog.CreateSchema, catalog.UpdateSchema](structcopy.ForceSend(schemaForceSend...)) func makeUpdateSchema(id, newName string, config *catalog.CreateSchema) catalog.UpdateSchema { diff --git a/bundle/direct/dresources/vector_search_endpoint.go b/bundle/direct/dresources/vector_search_endpoint.go index f1b45ac1198..c7b378319ad 100644 --- a/bundle/direct/dresources/vector_search_endpoint.go +++ b/bundle/direct/dresources/vector_search_endpoint.go @@ -62,6 +62,7 @@ func (*ResourceVectorSearchEndpoint) PrepareState(input *resources.VectorSearchE return &input.CreateEndpoint } +// UsagePolicyId is missing in remote. var toCreateVectorSearchEndpoint = newCopier[VectorSearchEndpointRemote, vectorsearch.CreateEndpoint]() func (*ResourceVectorSearchEndpoint) RemapState(remote *VectorSearchEndpointRemote) *vectorsearch.CreateEndpoint { diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 5d819520fc9..c25db42d928 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -66,7 +66,7 @@ func (r *ResourceVolume) DoCreate(ctx context.Context, config *VolumeState) (str var volumeForceSend = []string{"Comment"} // Name addresses the volume by its full name; the name in the config goes in NewName when it -// differs, which only DoUpdateWithID does. +// differs, which only DoUpdateWithID does. Owner is not supported by DABs. var toUpdateVolume = newCopier[VolumeState, catalog.UpdateVolumeRequestContent]( structcopy.Skip("Name"), structcopy.ForceSend(volumeForceSend...), From d6055147c6f7b6d44d357924827a4e111cb6622a Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 19 Nov 2026 01:13:29 +0000 Subject: [PATCH 4/9] direct: regenerate the field copy listing in generate-direct generate-direct-copies refreshes testdata/field_copies.txt and runs last in the generate-direct chain, so bump-sdk-post records the fields an SDK bump adds and the generated-files check covers the listing. Document newCopier in the resource guidelines, and reviewing the listing in the bump-sdk skill. Co-authored-by: Isaac --- .agents/rules/auto-generated-files.md | 6 +++-- .agents/skills/bump-sdk/SKILL.md | 9 ++++--- Taskfile.yml | 38 ++++++++++++++++++++------- bundle/direct/dresources/README.md | 33 +++++++++++++++++++++++ 4 files changed, 71 insertions(+), 15 deletions(-) diff --git a/.agents/rules/auto-generated-files.md b/.agents/rules/auto-generated-files.md index 6d29f4f55f9..bcd4ee58d85 100644 --- a/.agents/rules/auto-generated-files.md +++ b/.agents/rules/auto-generated-files.md @@ -14,6 +14,7 @@ globs: - "internal/genkit/tagging.py" - "internal/mocks/**/*.go" - "bundle/direct/dresources/configs/*.generated.yml" + - "bundle/direct/dresources/testdata/field_copies.txt" - "bundle/internal/validation/generated/*.go" - "bundle/schema/jsonschema.json" - "python/databricks/bundles/version.py" @@ -33,6 +34,7 @@ paths: - "internal/genkit/tagging.py" - "internal/mocks/**/*.go" - "bundle/direct/dresources/configs/*.generated.yml" + - "bundle/direct/dresources/testdata/field_copies.txt" - "bundle/internal/validation/generated/*.go" - "bundle/schema/jsonschema.json" - "python/databricks/bundles/version.py" @@ -63,8 +65,8 @@ Files matching this rule's glob pattern are most likely generated artifacts. Aut - OpenAPI SDK/CLI command stubs and related generated artifacts: - `./task generate-cligen` — regenerates `cmd/account/**`, `cmd/workspace/**` and `.gitattributes` from the checked-in `.codegen/cli.json`. - `./task generate-clijson` — refreshes `.codegen/cli.json` from the OpenAPI spec via genkit (requires universe repo); also updates `internal/genkit/tagging.py`. -- Direct engine generated YAML: - - `./task generate-direct` (or `./task generate-direct-apitypes`, `./task generate-direct-resources`) +- Direct engine generated YAML and field copy listing: + - `./task generate-direct` (or `./task generate-direct-apitypes`, `./task generate-direct-resources`, `./task generate-direct-copies`) - Bundle schemas: - `./task generate-schema` - Rewrites `bundle/internal/schema/annotations.yml` in place: upstream docs are sourced from `.codegen/cli.json` at generation time, and the file is synced with the config structure (placeholders added, stale entries dropped). diff --git a/.agents/skills/bump-sdk/SKILL.md b/.agents/skills/bump-sdk/SKILL.md index 232b5450f8b..f87043e63ac 100644 --- a/.agents/skills/bump-sdk/SKILL.md +++ b/.agents/skills/bump-sdk/SKILL.md @@ -9,7 +9,7 @@ allowed-tools: Read, Edit, Write, Bash, Glob, Grep, WebFetch, AskUserQuestion The SDK version lives in `go.mod` (`github.com/databricks/databricks-sdk-go`) and the pinned spec SHA lives in `.codegen/_openapi_sha`. These two move as a pair; everything else in this skill is regenerated from them or is fallout you fix by hand. -Do not hand-edit generated files (`.codegen/cli.json`, `cmd/workspace/*`, `cmd/account/*`, `bundle/schema/jsonschema.json`, `bundle/internal/validation/generated/*`, `bundle/direct/dresources/configs/*.generated.yml`, `python/databricks/bundles/**`); regenerate them. +Do not hand-edit generated files (`.codegen/cli.json`, `cmd/workspace/*`, `cmd/account/*`, `bundle/schema/jsonschema.json`, `bundle/internal/validation/generated/*`, `bundle/direct/dresources/configs/*.generated.yml`, `bundle/direct/dresources/testdata/field_copies.txt`, `python/databricks/bundles/**`); regenerate them. The Python tasks (`pydabs-*`, and the `pydabs-codegen` step inside `generate-check`) all run through `uv`. If one fails because `uv` is missing or because the host's `python3` is too old (e.g. 3.9), install `uv` (`curl -LsSf https://astral.sh/uv/install.sh | sh`) rather than touching the system Python: `uv run` provisions the interpreter each package pins (`>=3.10`, and `==3.13.*` under `python/codegen/`) and downloads it if needed. Do not chase the system Python version. The acceptance goldens are the exception: they need CI's `uv` index, see step 4. @@ -52,7 +52,7 @@ Decide with the user whether to keep the tagging-producer drift or restore those The task ends with step 4 (`generate-clijson-post` runs `bump-sdk-post`) and ignores its error, so read the end of its output, or go on with step 4 directly. **4. Regenerate everything downstream: `./task bump-sdk-post`.** -This is the whole mechanical part of the bump: `install-pythons` and `install-ruff`, then `generate-check` (command stubs, refschema, schema, validation, direct-engine YAML, pydabs), then `test-update` for the acceptance goldens. +This is the whole mechanical part of the bump: `install-pythons` and `install-ruff`, then `generate-check` (command stubs, refschema, schema, validation, direct-engine YAML and field copy listing, pydabs), then `test-update` for the acceptance goldens. Everything after it needs judgment. Run directly, it fails on errors: `generate-refschema` and `test-update` build the CLI, so it fails while the bump still breaks compilation; fix that (step 5) and rerun the task. @@ -62,13 +62,14 @@ Run it in CI's environment, or the goldens pick up local drift: - Ruff: `./task install-ruff` installs CI's version (0.9.1) via uv; `bump-sdk-post` adds uv's tool executable directory to `PATH` for `test-update`. The acceptance runner only checks that Ruff is available. - Go: the `toolchain` in `go.mod`. A newer local `go` also changes goldens and generated `gofmt` output; use `GOTOOLCHAIN=` if they differ. -Before going on, account for every changed file outside the generated set, including the bumper's (its environment is not CI's either): a bump without fallout changes `cmd/**`, `bundle/schema/jsonschema.json`, `bundle/internal/validation/generated/*`, `python/databricks/bundles/**`, `acceptance/bundle/refschema/output` and help-text goldens (`acceptance/help/output.txt`, `acceptance/cmd/account/account-help/output.txt`). +Before going on, account for every changed file outside the generated set, including the bumper's (its environment is not CI's either): a bump without fallout changes `cmd/**`, `bundle/schema/jsonschema.json`, `bundle/internal/validation/generated/*`, `bundle/direct/dresources/testdata/field_copies.txt`, `python/databricks/bundles/**`, `acceptance/bundle/refschema/output` and help-text goldens (`acceptance/help/output.txt`, `acceptance/cmd/account/account-help/output.txt`). Revert any golden change you can't attribute to the new SDK or spec. **5. Handle SDK breaking changes.** Read the SDK's `CHANGELOG.md` at the target version (in the module cache) to enumerate breaking changes before chasing compile errors. A removed struct field that the CLI used (e.g. `jobs.AiRuntimeTask.CodeSourcePath`) should have its usage temporarily disabled with a comment noting it returns in a later SDK bump, not deleted outright. -A new struct field triggers an `exhaustruct` lint failure in `bundle/direct/dresources/*`. Run `./task lint` and (in case of issues) wire the field through `PrepareState` and `RemapState` when it exists on both the input and remote types. +A new struct field usually needs no wiring in `bundle/direct/dresources/*`: the structs built with `newCopier` pick it up, and step 4 records that in `bundle/direct/dresources/testdata/field_copies.txt`. Read that file's diff as the review of the wiring: `copied` means a request or state now carries the field, `not in source` means it stays unset because the struct it is copied from has no such field (see "Copying fields between SDK structs" in `bundle/direct/dresources/README.md`). `TestFieldCopies` fails on a field whose type or JSON name differs between the two structs; skip it in the copier and set it by hand. +A struct literal that is still built by hand triggers an `exhaustruct` lint failure instead. Run `./task lint` and (in case of issues) wire the field through by hand. Where a remote is rebuilt by hand from a GET response (e.g. `makeJobRunRemote` maps `jobs.Run` into the `jobs.RunNow`-shaped `JobRunRemote`), copy the new field from the response if the response has it, and list it with the request-only zero values otherwise. `job_runs.yml` must list every `jobs.RunNow` field under `ignore_remote_changes`; `TestJobRunIgnoresEveryRequestField` fails on a new one. A field the new spec now annotates as output-only may already be emitted into `.generated.yml`, making the manual entry in `.yml` redundant; `TestResourcesYMLNoRedundantRules` catches this, so remove the manual entry. diff --git a/Taskfile.yml b/Taskfile.yml index 49bae2e2ef8..8a162dade5f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -996,8 +996,9 @@ tasks: # then refresh the acceptance goldens. generate-clijson-post runs it last; # run it directly to redo it, e.g. after hand fixes. generate-refschema and # test-update build the CLI, so it fails while the bump breaks compilation - # (code using a removed SDK field). Those fixes, exhaustruct wiring and the - # changelog PR link stay manual (.agents/skills/bump-sdk). + # (code using a removed SDK field). Those fixes, exhaustruct wiring of the + # struct literals that are built by hand and the changelog PR link stay + # manual (.agents/skills/bump-sdk). bump-sdk-post: desc: Regenerate everything downstream of an SDK bump and refresh acceptance goldens cmds: @@ -1066,13 +1067,14 @@ tasks: - "sh -c 'go run ./bundle/internal/validation/. && gofmt -w -s ./bundle/internal/validation/generated'" generate-direct: - desc: Generate direct engine config (apitypes + resources) - # Depend only on generate-direct-resources; it already pulls in - # generate-direct-apitypes transitively. Listing both as parallel deps - # races: resources reads apitypes.generated.yml while the apitypes step's - # `>` redirect is truncating it, silently dropping every resource whose - # type comes from that file. Keep this chain linear. - deps: ['generate-direct-resources'] + desc: Generate direct engine config (apitypes + resources) and field copy listing + # Depend only on generate-direct-copies; it already pulls in + # generate-direct-resources and generate-direct-apitypes transitively. + # Listing them as parallel deps races: resources reads + # apitypes.generated.yml while the apitypes step's `>` redirect is + # truncating it, silently dropping every resource whose type comes from + # that file. Keep this chain linear. + deps: ['generate-direct-copies'] generate-direct-apitypes: desc: Generate direct engine API types YAML @@ -1099,6 +1101,24 @@ tasks: cmds: - "uv run --script bundle/direct/tools/generate_resources.py .codegen/cli.json bundle/direct/dresources/apitypes.generated.yml bundle/direct/dresources/apitypes.yml acceptance/bundle/refschema/output bundle/direct/dresources/configs" + # Refreshes the listing of what every structcopy.Copier of the direct engine + # does with each field of the struct it builds (see TestFieldCopies). An SDK + # bump (go.mod / go.sum) that adds a field to such a struct changes it. Runs + # after generate-direct-resources, whose output the package embeds. + generate-direct-copies: + desc: Regenerate bundle/direct/dresources/testdata/field_copies.txt + deps: ['generate-direct-resources'] + sources: + - bundle/direct/dresources/**/*.go + - bundle/direct/dresources/configs/*.yml + - libs/structs/structcopy/*.go + - go.mod + - go.sum + generates: + - bundle/direct/dresources/testdata/field_copies.txt + cmds: + - go test ./bundle/direct/dresources -run '^TestFieldCopies$' -update + # pydabs-* tasks are defined in python/Taskfile.yml (included above). # --- Benchmarks --- diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index 373af02b13f..4967a112958 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -109,6 +109,39 @@ So when a field comes back under a different path than `StateType` uses: This makes the field present in `InputType`, `StateType`, and `RemoteType`, so it participates in normal drift detection and is no longer subject to the `missing_in_remote` suppression. +## Copying fields between SDK structs: newCopier + +The SDK describes a resource with several structs (a response, a create request, an update +request) that share most of their fields. Do not list those fields in a struct literal. Declare a +copier instead, which copies every field the two structs have under the same name: + +```go +var toSchemaState = newCopier[catalog.SchemaInfo, SchemaState]() + +func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *SchemaState { + return toSchemaState.Copy(info) +} +``` + +A field the SDK adds to both structs is then copied without a code change. A `RemapState` written +this way is the dumb copy the section above asks for. + +Tell the copier about the exceptions only, each with a comment that says why: + + - `structcopy.Skip("Owner")` leaves a field unset although the source has it. Use it for a field the request must not carry and for one that is set by hand after the copy (the ID, a value that needs converting). + - `structcopy.ForceSend("Comment")` sends a field even when it is zero, so that removing it from the config clears it. Only list fields verified against a real workspace, see `schemaForceSend`. + +Do not set `ForceSendFields` next to a copy. The copier derives it: the copied fields that the +source lists, followed by the `ForceSend` ones. The SDK rejects a name its struct does not have, +and the name of a field that was not copied would send that field's zero value. + +A field with the same name but another type or JSON name in the two structs is an error until it +is skipped. `TestFieldCopies` reports it, and records what every copier does with each field of +the struct it builds in `testdata/field_copies.txt`. `./task generate-direct` regenerates that +file. On an SDK bump, read its diff the way you would review the wiring of a new field: it shows +which requests and states pick the field up, and which leave it unset because their source has +no such field. + ## OverrideChangeDesc Use `OverrideChangeDesc` only as a last resort when the `.yml` settings cannot express the needed logic. Skipping an action with `change.Action = deployplan.Skip` in `OverrideChangeDesc` creates a silent no-op: the plan shows no change even if the user's config differs from remote. Document the skip reason clearly in both the comment and `change.Reason`. From 5c5598f8409ba8208e1516f491f98c07d4e5d759 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 19 Nov 2026 01:18:02 +0000 Subject: [PATCH 5/9] direct: drop ForceSendFields: nil from struct literals exhaustruct made every struct literal list ForceSendFields, nearly always as nil. Mark the field optional for the linter and delete the lines: a struct that copies another one gets its ForceSendFields from structcopy, and any other literal lists it only to force a zero value. The rest of the diff is gofmt realigning the literals. job_run.go keeps its explicit nils, which its comments refer to. Co-authored-by: Isaac --- .golangci.yaml | 5 +++ bundle/direct/dresources/app.go | 5 +-- bundle/direct/dresources/catalog.go | 5 +-- bundle/direct/dresources/dashboard.go | 4 -- bundle/direct/dresources/database_instance.go | 7 ++- bundle/direct/dresources/experiment.go | 5 +-- bundle/direct/dresources/external_location.go | 5 +-- bundle/direct/dresources/genie_space.go | 1 - bundle/direct/dresources/grants.go | 31 ++++++------- bundle/direct/dresources/job.go | 1 - .../direct/dresources/materialization_set.go | 3 -- bundle/direct/dresources/mcp_service.go | 44 +++++++++---------- .../dresources/model_provider_service.go | 28 +++++------- bundle/direct/dresources/model_service.go | 24 +++++----- .../dresources/model_serving_endpoint.go | 10 ++--- bundle/direct/dresources/permissions.go | 1 - bundle/direct/dresources/postgres_branch.go | 36 +++++++-------- bundle/direct/dresources/postgres_catalog.go | 13 +++--- bundle/direct/dresources/postgres_data_api.go | 22 +++++----- bundle/direct/dresources/postgres_database.go | 27 +++++------- bundle/direct/dresources/postgres_endpoint.go | 31 ++++++------- bundle/direct/dresources/postgres_project.go | 39 ++++++++-------- bundle/direct/dresources/postgres_role.go | 28 +++++------- .../dresources/postgres_snapshot_schedule.go | 8 ++-- .../dresources/postgres_synced_table.go | 11 +++-- bundle/direct/dresources/registered_model.go | 7 ++- bundle/direct/dresources/schema.go | 5 +-- bundle/direct/dresources/secret.go | 36 +++++++-------- bundle/direct/dresources/secret_scope.go | 2 - .../dresources/synced_database_table.go | 5 +-- .../dresources/vector_search_endpoint.go | 5 +-- 31 files changed, 197 insertions(+), 257 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index f05060c41ae..4ec07e8ff13 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -174,6 +174,11 @@ linters: - encoded-compare # Always replaces Equal() with JSONEq, but sometimes exact string match is wanted for JSON exhaustive: default-signifies-exhaustive: true + exhaustruct_v5: + # A struct that copies another one gets its ForceSendFields from + # structcopy, and any other literal lists it only to force a zero value. + optional-patterns: + - '.*#ForceSendFields$' exclusions: generated: lax presets: diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index a24b6678aaa..f38cde1da6f 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -138,9 +138,8 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *AppState) (string, * // For nil (omitted) or false, use no_compute=true (do not start compute). noCompute := config.Lifecycle == nil || config.Lifecycle.Started == nil || !*config.Lifecycle.Started request := apps.CreateAppRequest{ - App: appRequestBody(config.App), - NoCompute: noCompute, - ForceSendFields: nil, + App: appRequestBody(config.App), + NoCompute: noCompute, } retrier := retries.New[apps.App](retries.WithTimeout(15*time.Minute), retries.WithRetryFunc(shouldRetry)) diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index b7f66acf32c..1b24c89ec47 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -130,8 +130,7 @@ func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config func (r *ResourceCatalog) DoDelete(ctx context.Context, id string, _ *CatalogState) error { return r.client.Catalogs.Delete(ctx, catalog.DeleteCatalogRequest{ - Name: id, - Force: true, - ForceSendFields: nil, + Name: id, + Force: true, }) } diff --git a/bundle/direct/dresources/dashboard.go b/bundle/direct/dresources/dashboard.go index 602cb0931dc..b4dfe9228aa 100644 --- a/bundle/direct/dresources/dashboard.go +++ b/bundle/direct/dresources/dashboard.go @@ -264,8 +264,6 @@ func (r *ResourceDashboard) DoCreate(ctx context.Context, config *DashboardState Dashboard: dashboard, DatasetCatalog: config.DatasetCatalog, DatasetSchema: config.DatasetSchema, - - ForceSendFields: nil, }) } start := time.Now() @@ -351,8 +349,6 @@ func (r *ResourceDashboard) DoUpdate(ctx context.Context, id string, config *Das Dashboard: dashboard, DatasetCatalog: config.DatasetCatalog, DatasetSchema: config.DatasetSchema, - - ForceSendFields: nil, }) if err != nil { return nil, err diff --git a/bundle/direct/dresources/database_instance.go b/bundle/direct/dresources/database_instance.go index 3df0c4bb5e0..59b21a8fec6 100644 --- a/bundle/direct/dresources/database_instance.go +++ b/bundle/direct/dresources/database_instance.go @@ -62,9 +62,8 @@ func (d *ResourceDatabaseInstance) WaitAfterCreate(ctx context.Context, id strin func (d *ResourceDatabaseInstance) DoDelete(ctx context.Context, name string, _ *database.DatabaseInstance) error { return d.client.Database.DeleteDatabaseInstance(ctx, database.DeleteDatabaseInstanceRequest{ - Name: name, - Purge: true, //nolint:staticcheck // SA1019: purge is deprecated in the SDK; keep sending purge=true so the recorded delete request stays unchanged - Force: false, - ForceSendFields: nil, + Name: name, + Purge: true, //nolint:staticcheck // SA1019: purge is deprecated in the SDK; keep sending purge=true so the recorded delete request stays unchanged + Force: false, }) } diff --git a/bundle/direct/dresources/experiment.go b/bundle/direct/dresources/experiment.go index 539fba6d0fe..b4c3c5ab352 100644 --- a/bundle/direct/dresources/experiment.go +++ b/bundle/direct/dresources/experiment.go @@ -51,9 +51,8 @@ func (r *ResourceExperiment) DoCreate(ctx context.Context, config *ml.CreateExpe func (r *ResourceExperiment) DoUpdate(ctx context.Context, id string, config *ml.CreateExperiment, _ *PlanEntry) (*ml.Experiment, error) { updateReq := ml.UpdateExperiment{ - ExperimentId: id, - NewName: config.Name, - ForceSendFields: nil, + ExperimentId: id, + NewName: config.Name, } return nil, r.client.Experiments.UpdateExperiment(ctx, updateReq) diff --git a/bundle/direct/dresources/external_location.go b/bundle/direct/dresources/external_location.go index 58a3f9c8759..1b0426ecb06 100644 --- a/bundle/direct/dresources/external_location.go +++ b/bundle/direct/dresources/external_location.go @@ -89,8 +89,7 @@ func (r *ResourceExternalLocation) DoUpdateWithID(ctx context.Context, id string func (r *ResourceExternalLocation) DoDelete(ctx context.Context, id string, _ *catalog.CreateExternalLocation) error { return r.client.ExternalLocations.Delete(ctx, catalog.DeleteExternalLocationRequest{ - Name: id, - Force: true, - ForceSendFields: nil, + Name: id, + Force: true, }) } diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 30c8a07485f..294cd98c14b 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -48,7 +48,6 @@ func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources. space, err := r.client.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ SpaceId: id, IncludeSerializedSpace: true, // otherwise etag isn't returned - ForceSendFields: nil, }) if err != nil { return nil, err diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 005305804e5..78a2a10da24 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -150,7 +150,6 @@ func (r *ResourceGrants) DoUpdate(ctx context.Context, _ string, state *GrantsSt FullName: state.FullName, Changes: buildGrantChanges(state.EmbeddedSlice, remoteGrantPrivileges(entry)), OmitPermissionsInResponse: false, - ForceSendFields: nil, }) return nil, err } @@ -166,10 +165,9 @@ func buildGrantChanges(desiredAssignments []catalog.PrivilegeAssignment, remote for _, ga := range desiredAssignments { desiredPrincipals[ga.Principal] = struct{}{} changes = append(changes, catalog.PermissionsChange{ - Principal: ga.Principal, - Add: ga.Privileges, - Remove: grantRemovals(ga.Privileges, remote[ga.Principal]), - ForceSendFields: nil, + Principal: ga.Principal, + Add: ga.Privileges, + Remove: grantRemovals(ga.Privileges, remote[ga.Principal]), }) } @@ -183,10 +181,9 @@ func buildGrantChanges(desiredAssignments []catalog.PrivilegeAssignment, remote slices.Sort(removedPrincipals) for _, principal := range removedPrincipals { changes = append(changes, catalog.PermissionsChange{ - Principal: principal, - Add: nil, - Remove: grantRemovals(nil, remote[principal]), - ForceSendFields: nil, + Principal: principal, + Add: nil, + Remove: grantRemovals(nil, remote[principal]), }) } return changes @@ -244,12 +241,11 @@ func (r *ResourceGrants) listGrants(ctx context.Context, securableType, fullName pageToken := "" for { resp, err := r.client.Grants.Get(ctx, catalog.GetGrantRequest{ - FullName: fullName, - MaxResults: 0, - PageToken: pageToken, - Principal: "", - SecurableType: securableType, - ForceSendFields: nil, + FullName: fullName, + MaxResults: 0, + PageToken: pageToken, + Principal: "", + SecurableType: securableType, }) if err != nil { return nil, err @@ -259,9 +255,8 @@ func (r *ResourceGrants) listGrants(ctx context.Context, securableType, fullName continue } assignments = append(assignments, catalog.PrivilegeAssignment{ - Principal: assignment.Principal, - Privileges: assignment.Privileges, - ForceSendFields: nil, + Principal: assignment.Principal, + Privileges: assignment.Privileges, }) } if resp.NextPageToken == "" { diff --git a/bundle/direct/dresources/job.go b/bundle/direct/dresources/job.go index e0e7c3947fa..017552a00c2 100644 --- a/bundle/direct/dresources/job.go +++ b/bundle/direct/dresources/job.go @@ -124,7 +124,6 @@ func (r *ResourceJob) DoRead(ctx context.Context, id string) (*JobRemote, error) JobId: idInt, PageToken: "", IncludeTriggerState: false, - ForceSendFields: nil, }) if err != nil { return nil, err diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go index ce99d3cda5f..f1a54010d2b 100644 --- a/bundle/direct/dresources/materialization_set.go +++ b/bundle/direct/dresources/materialization_set.go @@ -183,7 +183,6 @@ func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.Ma operation, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ MaterializationSet: withLowercaseFeatureNames(*config), RequestId: uuid.New().String(), - ForceSendFields: nil, }) if err != nil { return "", nil, err @@ -266,7 +265,6 @@ func (r *ResourceMaterializationSet) DoUpdate(ctx context.Context, id string, co MaterializationSet: update, UpdateMask: *fieldmask.New([]string{"*"}), RequestId: uuid.New().String(), - ForceSendFields: nil, }) if err != nil { return nil, err @@ -286,7 +284,6 @@ func (r *ResourceMaterializationSet) DoDelete(ctx context.Context, id string, _ MaterializationSetId: id, RequestId: uuid.New().String(), AllowMissing: false, - ForceSendFields: nil, }) if err != nil { return err diff --git a/bundle/direct/dresources/mcp_service.go b/bundle/direct/dresources/mcp_service.go index b44309375e3..91294c7d132 100644 --- a/bundle/direct/dresources/mcp_service.go +++ b/bundle/direct/dresources/mcp_service.go @@ -55,11 +55,10 @@ func responseToMcpServiceConfig(ms *catalog.McpService) (*resources.McpServiceCo return nil, err } return &resources.McpServiceConfig{ - Parent: parent, - McpServiceId: id, - Comment: ms.Comment, - Config: ms.Config, - ForceSendFields: nil, + Parent: parent, + McpServiceId: id, + Comment: ms.Comment, + Config: ms.Config, }, nil } @@ -68,17 +67,16 @@ func responseToMcpServiceConfig(ms *catalog.McpService) (*resources.McpServiceCo // (server-derived) and sent as its zero value. func mcpServiceBody(config *resources.McpServiceConfig) catalog.McpService { return catalog.McpService{ - Comment: config.Comment, - Config: config.Config, - CreateTime: nil, - CreatedBy: "", - EffectiveOwner: "", - Etag: "", - MetastoreId: "", - Name: "", - UpdateTime: nil, - UpdatedBy: "", - ForceSendFields: nil, + Comment: config.Comment, + Config: config.Config, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + Etag: "", + MetastoreId: "", + Name: "", + UpdateTime: nil, + UpdatedBy: "", } } @@ -134,11 +132,10 @@ func (r *ResourceMcpService) DoCreate(ctx context.Context, config *resources.Mcp // concurrency on these resources. func (r *ResourceMcpService) DoUpdate(ctx context.Context, id string, config *resources.McpServiceConfig, _ *PlanEntry) (*resources.McpServiceConfig, error) { resp, err := r.client.AiGateway.UpdateMcpService(ctx, catalog.UpdateMcpServiceRequest{ - Etag: "", - McpService: mcpServiceBody(config), - Name: mcpServiceNamePrefix + id, - UpdateMask: fieldmask.FieldMask{Paths: []string{"*"}}, - ForceSendFields: nil, + Etag: "", + McpService: mcpServiceBody(config), + Name: mcpServiceNamePrefix + id, + UpdateMask: fieldmask.FieldMask{Paths: []string{"*"}}, }) if err != nil { return nil, err @@ -148,8 +145,7 @@ func (r *ResourceMcpService) DoUpdate(ctx context.Context, id string, config *re func (r *ResourceMcpService) DoDelete(ctx context.Context, id string, _ *resources.McpServiceConfig) error { return r.client.AiGateway.DeleteMcpService(ctx, catalog.DeleteMcpServiceRequest{ - Etag: "", - Name: mcpServiceNamePrefix + id, - ForceSendFields: nil, + Etag: "", + Name: mcpServiceNamePrefix + id, }) } diff --git a/bundle/direct/dresources/model_provider_service.go b/bundle/direct/dresources/model_provider_service.go index c71a1c30818..da569b72b90 100644 --- a/bundle/direct/dresources/model_provider_service.go +++ b/bundle/direct/dresources/model_provider_service.go @@ -59,7 +59,6 @@ func responseToModelProviderServiceConfig(ms *catalog.ModelProviderService) (*re ModelProviderServiceId: id, Comment: ms.Comment, Config: ms.Config, - ForceSendFields: nil, }, nil } @@ -68,17 +67,16 @@ func responseToModelProviderServiceConfig(ms *catalog.ModelProviderService) (*re // is OUTPUT_ONLY (server-derived) and sent as its zero value. func modelProviderServiceBody(config *resources.ModelProviderServiceConfig) catalog.ModelProviderService { return catalog.ModelProviderService{ - Comment: config.Comment, - Config: config.Config, - CreateTime: nil, - CreatedBy: "", - EffectiveOwner: "", - Etag: "", - MetastoreId: "", - Name: "", - UpdateTime: nil, - UpdatedBy: "", - ForceSendFields: nil, + Comment: config.Comment, + Config: config.Config, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + Etag: "", + MetastoreId: "", + Name: "", + UpdateTime: nil, + UpdatedBy: "", } } @@ -139,7 +137,6 @@ func (r *ResourceModelProviderService) DoUpdate(ctx context.Context, id string, Name: modelProviderServiceNamePrefix + id, ModelProviderService: modelProviderServiceBody(config), UpdateMask: fieldmask.FieldMask{Paths: []string{"*"}}, - ForceSendFields: nil, }) if err != nil { return nil, err @@ -149,8 +146,7 @@ func (r *ResourceModelProviderService) DoUpdate(ctx context.Context, id string, func (r *ResourceModelProviderService) DoDelete(ctx context.Context, id string, _ *resources.ModelProviderServiceConfig) error { return r.client.AiGateway.DeleteModelProviderService(ctx, catalog.DeleteModelProviderServiceRequest{ - Etag: "", - Name: modelProviderServiceNamePrefix + id, - ForceSendFields: nil, + Etag: "", + Name: modelProviderServiceNamePrefix + id, }) } diff --git a/bundle/direct/dresources/model_service.go b/bundle/direct/dresources/model_service.go index 7f6f8324aac..837e48f8eab 100644 --- a/bundle/direct/dresources/model_service.go +++ b/bundle/direct/dresources/model_service.go @@ -60,11 +60,10 @@ func responseToModelServiceConfig(ms *catalog.ModelService) (*resources.ModelSer return nil, err } return &resources.ModelServiceConfig{ - Parent: parent, - ModelServiceId: id, - Comment: ms.Comment, - Config: ms.Config, - ForceSendFields: nil, + Parent: parent, + ModelServiceId: id, + Comment: ms.Comment, + Config: ms.Config, }, nil } @@ -84,7 +83,6 @@ func modelServiceBody(config *resources.ModelServiceConfig) catalog.ModelService SupportedApiTypes: nil, UpdateTime: nil, UpdatedBy: "", - ForceSendFields: nil, } } @@ -140,11 +138,10 @@ func (r *ResourceModelService) DoCreate(ctx context.Context, config *resources.M // concurrency on these resources. func (r *ResourceModelService) DoUpdate(ctx context.Context, id string, config *resources.ModelServiceConfig, _ *PlanEntry) (*resources.ModelServiceConfig, error) { resp, err := r.client.AiGateway.UpdateModelService(ctx, catalog.UpdateModelServiceRequest{ - Etag: "", - ModelService: modelServiceBody(config), - Name: modelServiceNamePrefix + id, - UpdateMask: fieldmask.FieldMask{Paths: []string{"*"}}, - ForceSendFields: nil, + Etag: "", + ModelService: modelServiceBody(config), + Name: modelServiceNamePrefix + id, + UpdateMask: fieldmask.FieldMask{Paths: []string{"*"}}, }) if err != nil { return nil, err @@ -154,8 +151,7 @@ func (r *ResourceModelService) DoUpdate(ctx context.Context, id string, config * func (r *ResourceModelService) DoDelete(ctx context.Context, id string, _ *resources.ModelServiceConfig) error { return r.client.AiGateway.DeleteModelService(ctx, catalog.DeleteModelServiceRequest{ - Etag: "", - Name: modelServiceNamePrefix + id, - ForceSendFields: nil, + Etag: "", + Name: modelServiceNamePrefix + id, }) } diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index 81a4fbcd2b2..b71b03ed993 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -271,17 +271,15 @@ func diffTags(currentTags, desiredTags []serving.EndpointTag) (addTags []serving v, ok := currentTagsMap[key] if !ok { addTags = append(addTags, serving.EndpointTag{ - Key: key, - Value: desiredValue, - ForceSendFields: nil, + Key: key, + Value: desiredValue, }) continue } if v != desiredValue { addTags = append(addTags, serving.EndpointTag{ - Key: key, - Value: desiredValue, - ForceSendFields: nil, + Key: key, + Value: desiredValue, }) } } diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 7f2eb0dbf04..47d75b75568 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -261,7 +261,6 @@ func (r *ResourcePermissions) DoUpdate(ctx context.Context, _ string, newState * UserName: p.UserName, ServicePrincipalName: p.ServicePrincipalName, GroupName: p.GroupName, - ForceSendFields: nil, } } diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index babdf14a0b3..c4a8e2b93cf 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -116,17 +116,15 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB Spec: &config.BranchSpec, // Output-only fields. - BranchId: "", - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - Uid: "", - UpdateTime: nil, - ForceSendFields: nil, + BranchId: "", + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + Uid: "", + UpdateTime: nil, }, ReplaceExisting: config.ReplaceExisting != nil && *config.ReplaceExisting, - ForceSendFields: nil, } start := time.Now() waiter, err := r.client.Postgres.CreateBranch(ctx, req) @@ -284,14 +282,13 @@ func (r *ResourcePostgresBranch) DoUpdate(ctx context.Context, id string, config Spec: &config.BranchSpec, // Output-only fields. - BranchId: "", - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - Uid: "", - UpdateTime: nil, - ForceSendFields: nil, + BranchId: "", + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + Uid: "", + UpdateTime: nil, }, Name: id, UpdateMask: fieldmask.FieldMask{ @@ -312,9 +309,8 @@ func (r *ResourcePostgresBranch) DoUpdate(ctx context.Context, id string, config func (r *ResourcePostgresBranch) DoDelete(ctx context.Context, id string, state *PostgresBranchState) error { waiter, err := r.client.Postgres.DeleteBranch(ctx, postgres.DeleteBranchRequest{ - Name: id, - Purge: state.PurgeOnDelete != nil && *state.PurgeOnDelete, - ForceSendFields: nil, + Name: id, + Purge: state.PurgeOnDelete != nil && *state.PurgeOnDelete, }) if err != nil { return err diff --git a/bundle/direct/dresources/postgres_catalog.go b/bundle/direct/dresources/postgres_catalog.go index e2b32665d85..2c7c27f31e6 100644 --- a/bundle/direct/dresources/postgres_catalog.go +++ b/bundle/direct/dresources/postgres_catalog.go @@ -98,13 +98,12 @@ func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *Postgres Spec: &config.CatalogCatalogSpec, // Output-only fields. - CatalogId: "", - CreateTime: nil, - Name: "", - Status: nil, - Uid: "", - UpdateTime: nil, - ForceSendFields: nil, + CatalogId: "", + CreateTime: nil, + Name: "", + Status: nil, + Uid: "", + UpdateTime: nil, }, }) if err != nil { diff --git a/bundle/direct/dresources/postgres_data_api.go b/bundle/direct/dresources/postgres_data_api.go index ba94134e0ca..25f0481fa20 100644 --- a/bundle/direct/dresources/postgres_data_api.go +++ b/bundle/direct/dresources/postgres_data_api.go @@ -123,12 +123,11 @@ func (r *ResourcePostgresDataApi) DoCreate(ctx context.Context, config *Postgres Spec: &config.DataApiDataApiSpec, // Output-only fields; the parent is carried in the request path. - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - UpdateTime: nil, - ForceSendFields: nil, + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, }, }) if err != nil { @@ -151,12 +150,11 @@ func (r *ResourcePostgresDataApi) DoUpdate(ctx context.Context, id string, confi Spec: &config.DataApiDataApiSpec, // Output-only fields; the name is carried in the request path. - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - UpdateTime: nil, - ForceSendFields: nil, + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, }, UpdateMask: fieldmask.FieldMask{ Paths: dataApiUpdateMask, diff --git a/bundle/direct/dresources/postgres_database.go b/bundle/direct/dresources/postgres_database.go index 034c7532eac..c1daf61488d 100644 --- a/bundle/direct/dresources/postgres_database.go +++ b/bundle/direct/dresources/postgres_database.go @@ -105,16 +105,14 @@ func (r *ResourcePostgresDatabase) DoCreate(ctx context.Context, config *Postgre Spec: &config.DatabaseDatabaseSpec, // Output-only fields. - DatabaseId: "", - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - UpdateTime: nil, - ForceSendFields: nil, + DatabaseId: "", + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, }, ReplaceExisting: config.ReplaceExisting != nil && *config.ReplaceExisting, - ForceSendFields: nil, }) if err != nil { return "", nil, err @@ -142,13 +140,12 @@ func (r *ResourcePostgresDatabase) DoUpdate(ctx context.Context, id string, conf Spec: &config.DatabaseDatabaseSpec, // Output-only fields. - DatabaseId: "", - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - UpdateTime: nil, - ForceSendFields: nil, + DatabaseId: "", + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, }, Name: id, UpdateMask: fieldmask.FieldMask{ diff --git a/bundle/direct/dresources/postgres_endpoint.go b/bundle/direct/dresources/postgres_endpoint.go index 188cb44c216..29cc1669a51 100644 --- a/bundle/direct/dresources/postgres_endpoint.go +++ b/bundle/direct/dresources/postgres_endpoint.go @@ -145,17 +145,15 @@ func (r *ResourcePostgresEndpoint) DoCreate(ctx context.Context, config *Postgre Spec: &config.EndpointSpec, // Output-only fields. - EndpointId: "", - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - Uid: "", - UpdateTime: nil, - ForceSendFields: nil, + EndpointId: "", + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + Uid: "", + UpdateTime: nil, }, ReplaceExisting: config.ReplaceExisting != nil && *config.ReplaceExisting, - ForceSendFields: nil, }) if err != nil { return "", nil, err @@ -195,14 +193,13 @@ func (r *ResourcePostgresEndpoint) DoUpdate(ctx context.Context, id string, conf Spec: &config.EndpointSpec, // Output-only fields. - EndpointId: "", - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - Uid: "", - UpdateTime: nil, - ForceSendFields: nil, + EndpointId: "", + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + Uid: "", + UpdateTime: nil, }, Name: id, UpdateMask: fieldmask.FieldMask{ diff --git a/bundle/direct/dresources/postgres_project.go b/bundle/direct/dresources/postgres_project.go index 8caf845b0e1..64626301952 100644 --- a/bundle/direct/dresources/postgres_project.go +++ b/bundle/direct/dresources/postgres_project.go @@ -113,15 +113,14 @@ func (r *ResourcePostgresProject) DoCreate(ctx context.Context, config *Postgres InitialEndpointSpec: nil, // Output-only fields. - ProjectId: "", - CreateTime: nil, - DeleteTime: nil, - Name: "", - PurgeTime: nil, - Status: nil, - Uid: "", - UpdateTime: nil, - ForceSendFields: nil, + ProjectId: "", + CreateTime: nil, + DeleteTime: nil, + Name: "", + PurgeTime: nil, + Status: nil, + Uid: "", + UpdateTime: nil, }, } start := time.Now() @@ -235,15 +234,14 @@ func (r *ResourcePostgresProject) DoUpdate(ctx context.Context, id string, confi InitialEndpointSpec: nil, // Output-only fields. - ProjectId: "", - CreateTime: nil, - DeleteTime: nil, - Name: "", - PurgeTime: nil, - Status: nil, - Uid: "", - UpdateTime: nil, - ForceSendFields: nil, + ProjectId: "", + CreateTime: nil, + DeleteTime: nil, + Name: "", + PurgeTime: nil, + Status: nil, + Uid: "", + UpdateTime: nil, }, Name: id, UpdateMask: fieldmask.FieldMask{ @@ -264,9 +262,8 @@ func (r *ResourcePostgresProject) DoUpdate(ctx context.Context, id string, confi func (r *ResourcePostgresProject) DoDelete(ctx context.Context, id string, state *PostgresProjectState) error { waiter, err := r.client.Postgres.DeleteProject(ctx, postgres.DeleteProjectRequest{ - Name: id, - Purge: state.PurgeOnDelete != nil && *state.PurgeOnDelete, - ForceSendFields: nil, + Name: id, + Purge: state.PurgeOnDelete != nil && *state.PurgeOnDelete, }) if err != nil { return err diff --git a/bundle/direct/dresources/postgres_role.go b/bundle/direct/dresources/postgres_role.go index 3e630fa95d7..47ba9bdd5a6 100644 --- a/bundle/direct/dresources/postgres_role.go +++ b/bundle/direct/dresources/postgres_role.go @@ -134,16 +134,14 @@ func (r *ResourcePostgresRole) DoCreate(ctx context.Context, config *PostgresRol Spec: &config.RoleRoleSpec, // Output-only fields. - RoleId: "", - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - UpdateTime: nil, - ForceSendFields: nil, + RoleId: "", + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, }, ReplaceExisting: config.ReplaceExisting != nil && *config.ReplaceExisting, - ForceSendFields: nil, } start := time.Now() waiter, err := r.client.Postgres.CreateRole(ctx, req) @@ -212,13 +210,12 @@ func (r *ResourcePostgresRole) DoUpdate(ctx context.Context, id string, config * Spec: &config.RoleRoleSpec, // Output-only fields. - RoleId: "", - CreateTime: nil, - Name: "", - Parent: "", - Status: nil, - UpdateTime: nil, - ForceSendFields: nil, + RoleId: "", + CreateTime: nil, + Name: "", + Parent: "", + Status: nil, + UpdateTime: nil, }, UpdateMask: fieldmask.FieldMask{ Paths: fieldPaths, @@ -243,7 +240,6 @@ func (r *ResourcePostgresRole) DoDelete(ctx context.Context, id string, _ *Postg // user-facing config we don't expose, and it would spin up compute to // run reassignment SQL. ReassignOwnedTo: "", - ForceSendFields: nil, }) if err != nil { return err diff --git a/bundle/direct/dresources/postgres_snapshot_schedule.go b/bundle/direct/dresources/postgres_snapshot_schedule.go index 3f73e8c729f..90714e8bc52 100644 --- a/bundle/direct/dresources/postgres_snapshot_schedule.go +++ b/bundle/direct/dresources/postgres_snapshot_schedule.go @@ -59,9 +59,8 @@ func (*ResourcePostgresSnapshotSchedule) PrepareState(input *resources.PostgresS func (*ResourcePostgresSnapshotSchedule) RemapState(remote *PostgresSnapshotScheduleRemote) *PostgresSnapshotScheduleState { return &PostgresSnapshotScheduleState{ - Branch: remote.Branch, - Schedule: remote.Schedule, - ForceSendFields: nil, + Branch: remote.Branch, + Schedule: remote.Schedule, } } @@ -96,8 +95,7 @@ func (r *ResourcePostgresSnapshotSchedule) updateSnapshotSchedule(ctx context.Co Schedule: cadences, // Name is carried in the request's Name field, not the body. - Name: "", - ForceSendFields: nil, + Name: "", }, UpdateMask: fieldmask.FieldMask{ Paths: []string{"schedule"}, diff --git a/bundle/direct/dresources/postgres_synced_table.go b/bundle/direct/dresources/postgres_synced_table.go index a94ba9b7c19..e69bf4d0684 100644 --- a/bundle/direct/dresources/postgres_synced_table.go +++ b/bundle/direct/dresources/postgres_synced_table.go @@ -105,12 +105,11 @@ func (r *ResourcePostgresSyncedTable) DoCreate(ctx context.Context, config *Post Spec: &config.SyncedTableSyncedTableSpec, // Output-only fields. - SyncedTableId: "", - CreateTime: nil, - Name: "", - Status: nil, - Uid: "", - ForceSendFields: nil, + SyncedTableId: "", + CreateTime: nil, + Name: "", + Status: nil, + Uid: "", }, }) if err != nil { diff --git a/bundle/direct/dresources/registered_model.go b/bundle/direct/dresources/registered_model.go index cb1e65a5769..eff7fb3855b 100644 --- a/bundle/direct/dresources/registered_model.go +++ b/bundle/direct/dresources/registered_model.go @@ -39,10 +39,9 @@ func (*ResourceRegisteredModel) RemapState(model *catalog.RegisteredModelInfo) * func (r *ResourceRegisteredModel) DoRead(ctx context.Context, id string) (*catalog.RegisteredModelInfo, error) { return r.client.RegisteredModels.Get(ctx, catalog.GetRegisteredModelRequest{ - FullName: id, - IncludeAliases: true, - IncludeBrowse: false, - ForceSendFields: nil, + FullName: id, + IncludeAliases: true, + IncludeBrowse: false, }) } diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 2bb296be75d..5813459dd74 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -132,8 +132,7 @@ func (*ResourceSchema) IDFromState(state *SchemaState) string { func (r *ResourceSchema) DoDelete(ctx context.Context, id string, _ *SchemaState) error { return r.client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{ - FullName: id, - Force: true, - ForceSendFields: nil, + FullName: id, + Force: true, }) } diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 28c83d5a68f..367e012bbbc 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -25,22 +25,21 @@ func (*ResourceSecret) New(client *databricks.WorkspaceClient) *ResourceSecret { func (*ResourceSecret) PrepareState(input *resources.Secret) *catalog.Secret { return &catalog.Secret{ - CatalogName: input.CatalogName, - SchemaName: input.SchemaName, - Name: input.Name, - Value: input.Value, - Comment: input.Comment, - ExpireTime: input.ExpireTime, - Owner: "", - CreateTime: nil, - CreatedBy: "", - EffectiveOwner: "", - EffectiveValue: "", - FullName: "", - MetastoreId: "", - UpdateTime: nil, - UpdatedBy: "", - ForceSendFields: nil, + CatalogName: input.CatalogName, + SchemaName: input.SchemaName, + Name: input.Name, + Value: input.Value, + Comment: input.Comment, + ExpireTime: input.ExpireTime, + Owner: "", + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", } } @@ -60,9 +59,8 @@ func remapSecretRemote(remote *catalog.Secret) *catalog.Secret { // recover the stored value from EffectiveValue. func (r *ResourceSecret) DoRead(ctx context.Context, id string) (*catalog.Secret, error) { remote, err := r.client.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ - FullName: id, - IncludeValue: true, - ForceSendFields: nil, + FullName: id, + IncludeValue: true, }) if err != nil { return nil, err diff --git a/bundle/direct/dresources/secret_scope.go b/bundle/direct/dresources/secret_scope.go index 4e9936025ca..68692dba3b0 100644 --- a/bundle/direct/dresources/secret_scope.go +++ b/bundle/direct/dresources/secret_scope.go @@ -54,7 +54,6 @@ func (*ResourceSecretScope) PrepareState(input *resources.SecretScope) *SecretSc ScopeBackendType: input.BackendType, BackendAzureKeyvault: input.KeyvaultMetadata, InitialManagePrincipal: "", - ForceSendFields: nil, } } @@ -80,7 +79,6 @@ func (r *ResourceSecretScope) DoRead(ctx context.Context, id string) (*SecretSco Scope: scope.Name, ScopeBackendType: scope.BackendType, BackendAzureKeyvault: scope.KeyvaultMetadata, - ForceSendFields: nil, }, nil } } diff --git a/bundle/direct/dresources/synced_database_table.go b/bundle/direct/dresources/synced_database_table.go index e1c0df12f00..faf003d46cc 100644 --- a/bundle/direct/dresources/synced_database_table.go +++ b/bundle/direct/dresources/synced_database_table.go @@ -36,8 +36,7 @@ func (r *ResourceSyncedDatabaseTable) DoCreate(ctx context.Context, config *data func (r *ResourceSyncedDatabaseTable) DoDelete(ctx context.Context, id string, _ *database.SyncedDatabaseTable) error { return r.client.Database.DeleteSyncedDatabaseTable(ctx, database.DeleteSyncedDatabaseTableRequest{ - Name: id, - PurgeData: false, - ForceSendFields: nil, + Name: id, + PurgeData: false, }) } diff --git a/bundle/direct/dresources/vector_search_endpoint.go b/bundle/direct/dresources/vector_search_endpoint.go index c7b378319ad..d06818dc457 100644 --- a/bundle/direct/dresources/vector_search_endpoint.go +++ b/bundle/direct/dresources/vector_search_endpoint.go @@ -137,9 +137,8 @@ func (r *ResourceVectorSearchEndpoint) DoUpdate(ctx context.Context, id string, if entry.Changes.HasChange(pathTargetQps) { _, err := r.client.VectorSearchEndpoints.PatchEndpoint(ctx, vectorsearch.PatchEndpointRequest{ - EndpointName: id, - TargetQps: config.TargetQps, - ForceSendFields: nil, + EndpointName: id, + TargetQps: config.TargetQps, }) if err != nil { return nil, err From 21714e0959e5195bf611b25fef32bcbe343872d7 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 19 Nov 2026 01:25:47 +0000 Subject: [PATCH 6/9] direct: copy dashboard and Genie space structs with structcopy Their content field is a string in the API and any in the config, so each copier skips it and the caller sets it. The force-send rules that were spliced into ForceSendFields by hand become ForceSend options. DoRead of a dashboard built the same state as responseToState, so it calls it. As with the other resources, a request no longer force-sends the zero value of a field it leaves unset when the state lists that field: the output-only fields of a dashboard (create_time, path, ...) and the etag of a Genie space update. A dashboard state no longer lists SerializedDashboard in ForceSendFields, which has no effect on a field of type any. Co-authored-by: Isaac --- bundle/direct/dresources/dashboard.go | 138 ++++++------------ bundle/direct/dresources/genie_space.go | 67 ++++----- .../dresources/testdata/field_copies.txt | 57 ++++++++ 3 files changed, 132 insertions(+), 130 deletions(-) diff --git a/bundle/direct/dresources/dashboard.go b/bundle/direct/dresources/dashboard.go index b4dfe9228aa..ff68804cbe6 100644 --- a/bundle/direct/dresources/dashboard.go +++ b/bundle/direct/dresources/dashboard.go @@ -4,15 +4,14 @@ import ( "context" "fmt" "path" - "slices" "strings" "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/marshal" @@ -118,39 +117,7 @@ func (r *ResourceDashboard) DoRead(ctx context.Context, id string) (*DashboardSt // don't spuriously republish. See DashboardState.Published. published := publishedErr == nil && publishedIsCurrent(dashboard, publishedDashboard) - forceSendFields := utils.FilterFields[DashboardState](dashboard.ForceSendFields) - // EmbedCredentials must always be included in ForceSendFields to ensure it's serialized - // even when false (zero value). - if !slices.Contains(forceSendFields, "EmbedCredentials") { - forceSendFields = append(forceSendFields, "EmbedCredentials") - } - - var embedCredentials bool - if publishedDashboard != nil { - embedCredentials = publishedDashboard.EmbedCredentials - } - - return &DashboardState{ - DisplayName: dashboard.DisplayName, - Etag: dashboard.Etag, - WarehouseId: dashboard.WarehouseId, - SerializedDashboard: dashboard.SerializedDashboard, - ParentPath: ensureWorkspacePrefix(dashboard.ParentPath), - // diffs are detected via etags, which will change if dataset_catalog/dataset_schema is updated. - DatasetCatalog: "", - DatasetSchema: "", - - // Output only fields. - CreateTime: dashboard.CreateTime, - DashboardId: dashboard.DashboardId, - LifecycleState: dashboard.LifecycleState, - Path: dashboard.Path, - UpdateTime: dashboard.UpdateTime, - ForceSendFields: forceSendFields, - - EmbedCredentials: embedCredentials, - Published: published, - }, nil + return responseToState(dashboard, publishedDashboard, dashboard.SerializedDashboard, published), nil } // publishedIsCurrent reports whether the published revision reflects the current @@ -176,22 +143,37 @@ func publishedIsCurrent(dashboard *dashboards.Dashboard, published *dashboards.P return !revisionTime.Before(updateTime) } +// serialized_dashboard is a string in the API and any in the config, so each copier skips it +// and the caller sets it. +var ( + // The output-only fields are not sent. Fields like "embed_credentials" are part of the + // bundle configuration but not the create request here, so they are left out. + toDashboardRequest = newCopier[DashboardState, dashboards.Dashboard]( + structcopy.Skip("CreateTime", "DashboardId", "LifecycleState", "Path", "UpdateTime", "SerializedDashboard"), + ) + + // embed_credentials as a zero valued default in resourcemutator/resource_mutator.go. + // Thus we always need to include it in the ForceSendFields list to ensure that it is sent to the server. + // DashboardId addresses the dashboard, so it is the id. + toPublishRequest = newCopier[DashboardState, dashboards.PublishRequest]( + structcopy.Skip("DashboardId"), + structcopy.ForceSend("EmbedCredentials"), + ) + + // The output-only fields (create_time, path, ...) are copied as well. dataset_catalog and + // dataset_schema are not reported: diffs are detected via etags, which will change if + // dataset_catalog/dataset_schema is updated. + // + // EmbedCredentials must always be included in ForceSendFields to ensure it's serialized + // even when false (zero value). + toDashboardState = newCopier[dashboards.Dashboard, DashboardState]( + structcopy.Skip("SerializedDashboard"), + structcopy.ForceSend("EmbedCredentials"), + ) +) + func prepareDashboardRequest(config *DashboardState) (dashboards.Dashboard, error) { - dashboard := dashboards.Dashboard{ - DisplayName: config.DisplayName, - ParentPath: config.ParentPath, - WarehouseId: config.WarehouseId, - Etag: config.Etag, - CreateTime: "", - DashboardId: "", - LifecycleState: "", - Path: "", - SerializedDashboard: "", - UpdateTime: "", - // Fields like "embed_credentials" are part of the bundle configuration but not the create request here. - // Thus we need to filter such fields out. - ForceSendFields: utils.FilterFields[dashboards.Dashboard](config.ForceSendFields), - } + dashboard := toDashboardRequest.Copy(config) switch v := config.SerializedDashboard.(type) { case nil: case string: @@ -199,58 +181,26 @@ func prepareDashboardRequest(config *DashboardState) (dashboards.Dashboard, erro default: return dashboards.Dashboard{}, fmt.Errorf("internal error: serialized_dashboard should have been normalized to a string, got %T", v) } - return dashboard, nil + return *dashboard, nil } func (r *ResourceDashboard) publishDashboard(ctx context.Context, id string, config *DashboardState) (*dashboards.PublishedDashboard, error) { - // embed_credentials as a zero valued default in resourcemutator/resource_mutator.go. - // Thus we always need to include it in the ForceSendFields list to ensure that it is sent to the server. - forceSendFields := utils.FilterFields[dashboards.PublishRequest](config.ForceSendFields) - if !slices.Contains(forceSendFields, "EmbedCredentials") { - forceSendFields = append(forceSendFields, "EmbedCredentials") - } - - return r.client.Lakeview.Publish(ctx, dashboards.PublishRequest{ - DashboardId: id, - EmbedCredentials: config.EmbedCredentials, - WarehouseId: config.WarehouseId, - ForceSendFields: forceSendFields, - }) + request := toPublishRequest.Copy(config) + request.DashboardId = id + return r.client.Lakeview.Publish(ctx, *request) } +// responseToState returns the state of a dashboard the API reported. publishResp is nil for a +// dashboard that is not published. func responseToState(createOrUpdateResp *dashboards.Dashboard, publishResp *dashboards.PublishedDashboard, serializedDashboard string, published bool) *DashboardState { - forceSendFields := utils.FilterFields[DashboardState](createOrUpdateResp.ForceSendFields) - // EmbedCredentials must always be included in ForceSendFields to ensure it's serialized - // even when false (zero value). - if !slices.Contains(forceSendFields, "EmbedCredentials") { - forceSendFields = append(forceSendFields, "EmbedCredentials") - } - - var embedCredentials bool + state := toDashboardState.Copy(createOrUpdateResp) + state.SerializedDashboard = serializedDashboard + state.ParentPath = ensureWorkspacePrefix(createOrUpdateResp.ParentPath) if publishResp != nil { - embedCredentials = publishResp.EmbedCredentials - } - - return &DashboardState{ - DisplayName: createOrUpdateResp.DisplayName, - Etag: createOrUpdateResp.Etag, - WarehouseId: createOrUpdateResp.WarehouseId, - SerializedDashboard: serializedDashboard, - ParentPath: ensureWorkspacePrefix(createOrUpdateResp.ParentPath), - DatasetCatalog: "", - DatasetSchema: "", - - // Output only fields - CreateTime: createOrUpdateResp.CreateTime, - DashboardId: createOrUpdateResp.DashboardId, - LifecycleState: createOrUpdateResp.LifecycleState, - Path: createOrUpdateResp.Path, - UpdateTime: createOrUpdateResp.UpdateTime, - ForceSendFields: forceSendFields, - - EmbedCredentials: embedCredentials, - Published: published, + state.EmbedCredentials = publishResp.EmbedCredentials } + state.Published = published + return state } func (r *ResourceDashboard) DoCreate(ctx context.Context, config *DashboardState) (string, *DashboardState, error) { diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 294cd98c14b..7d5c9d19217 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -5,13 +5,12 @@ import ( "errors" "fmt" "net/http" - "slices" "strings" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structcopy" "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -70,22 +69,32 @@ func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error } } -func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { +// serialized_space is a string in the API and any in the config, so each copier skips it and +// the caller sets it. +var ( // SerializedSpace is excluded so it is never force-sent as an empty value; it is always // populated from the bundle config on write. Done here (rather than in a RemapState) so // every remote-producing path (DoRead/DoCreate/DoUpdate) yields the same force-send set. - forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields, "SerializedSpace") - - return &resources.GenieSpaceConfig{ - Description: space.Description, - Etag: space.Etag, - Title: space.Title, - WarehouseId: space.WarehouseId, - ParentPath: ensureWorkspacePrefix(space.ParentPath), - SerializedSpace: serializedSpace, + toGenieSpaceConfig = newCopier[dashboards.GenieSpace, resources.GenieSpaceConfig]( + structcopy.Skip("SerializedSpace"), + ) + + toGenieCreateSpace = newCopier[resources.GenieSpaceConfig, dashboards.GenieCreateSpaceRequest]( + structcopy.Skip("SerializedSpace"), + ) + + // Etag is left empty, see DoUpdate. + toGenieUpdateSpace = newCopier[resources.GenieSpaceConfig, dashboards.GenieUpdateSpaceRequest]( + structcopy.Skip("SerializedSpace", "Etag"), + structcopy.ForceSend(genieSpaceForceSend...), + ) +) - ForceSendFields: forceSendFields, - } +func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { + config := toGenieSpaceConfig.Copy(space) + config.ParentPath = ensureWorkspacePrefix(space.ParentPath) + config.SerializedSpace = serializedSpace + return config } // isMissingGenieParentPathError reports whether the given Create error means @@ -129,17 +138,10 @@ func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.Gen return "", nil, err } - req := dashboards.GenieCreateSpaceRequest{ - Description: config.Description, - Title: config.Title, - WarehouseId: config.WarehouseId, - ParentPath: config.ParentPath, - SerializedSpace: serializedSpace, - - ForceSendFields: utils.FilterFields[dashboards.GenieCreateSpaceRequest](config.ForceSendFields), - } + req := toGenieCreateSpace.Copy(config) + req.SerializedSpace = serializedSpace - createResp, err := r.client.Genie.CreateSpace(ctx, req) + createResp, err := r.client.Genie.CreateSpace(ctx, *req) // Retry once after creating the parent directory when the workspace folder // is missing. Genie can surface this either as a standard missing-resource @@ -150,7 +152,7 @@ func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.Gen if err != nil { return "", nil, fmt.Errorf("failed to create parent directory: %w", err) } - createResp, err = r.client.Genie.CreateSpace(ctx, req) + createResp, err = r.client.Genie.CreateSpace(ctx, *req) } if err != nil { return "", nil, err @@ -182,17 +184,10 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re // migrates serialized_space to a newer schema version, so a stale etag would // 409 a legitimate update after a migration. Etag is therefore left empty; // out-of-band drift is surfaced on read via OverrideChangeDesc instead. - updateResp, err := r.client.Genie.UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ - SpaceId: id, - Description: config.Description, - Title: config.Title, - WarehouseId: config.WarehouseId, - ParentPath: config.ParentPath, - SerializedSpace: serializedSpace, - Etag: "", - - ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](append(slices.Clone(genieSpaceForceSend), config.ForceSendFields...)), - }) + req := toGenieUpdateSpace.Copy(config) + req.SpaceId = id + req.SerializedSpace = serializedSpace + updateResp, err := r.client.Genie.UpdateSpace(ctx, *req) if err != nil { return nil, err } diff --git a/bundle/direct/dresources/testdata/field_copies.txt b/bundle/direct/dresources/testdata/field_copies.txt index 3de36791e59..a2884048cf8 100644 --- a/bundle/direct/dresources/testdata/field_copies.txt +++ b/bundle/direct/dresources/testdata/field_copies.txt @@ -261,6 +261,30 @@ compute.Policy -> compute.CreatePolicy PolicyFamilyDefinitionOverrides: copied PolicyFamilyId: copied +dashboards.Dashboard -> dresources.DashboardState + CreateTime: copied + DashboardId: copied + DatasetCatalog: not in source + DatasetSchema: not in source + DisplayName: copied + EmbedCredentials: not in source, always sent + Etag: copied + LifecycleState: copied + ParentPath: copied + Path: copied + Published: not in source + SerializedDashboard: skipped + UpdateTime: copied + WarehouseId: copied + +dashboards.GenieSpace -> resources.GenieSpaceConfig + Description: copied + Etag: copied + ParentPath: copied + SerializedSpace: skipped + Title: copied + WarehouseId: copied + dresources.CatalogState -> catalog.UpdateCatalog Comment: copied, always sent CustomMaxRetentionHours: copied, always sent @@ -317,6 +341,23 @@ dresources.ClusterState -> compute.ResizeCluster ClusterId: not in source NumWorkers: copied +dresources.DashboardState -> dashboards.Dashboard + CreateTime: skipped + DashboardId: skipped + DisplayName: copied + Etag: copied + LifecycleState: skipped + ParentPath: copied + Path: skipped + SerializedDashboard: skipped + UpdateTime: skipped + WarehouseId: copied + +dresources.DashboardState -> dashboards.PublishRequest + DashboardId: skipped + EmbedCredentials: copied, always sent + WarehouseId: copied + dresources.MlflowModelRemote -> ml.CreateModelRequest Description: copied Name: copied @@ -565,6 +606,22 @@ pipelines.PipelineSpec -> pipelines.CreatePipeline Trigger: copied UsagePolicyId: copied +resources.GenieSpaceConfig -> dashboards.GenieCreateSpaceRequest + Description: copied + ParentPath: copied + SerializedSpace: skipped + Title: copied + WarehouseId: copied + +resources.GenieSpaceConfig -> dashboards.GenieUpdateSpaceRequest + Description: copied, always sent + Etag: skipped + ParentPath: copied + SerializedSpace: skipped + SpaceId: not in source + Title: copied + WarehouseId: copied + serving.AiGatewayConfig -> serving.PutAiGatewayRequest FallbackConfig: copied Guardrails: copied From bd2077f2b7225cc4c2f8a68d3f280cb879506eb6 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 19 Nov 2026 01:49:06 +0000 Subject: [PATCH 7/9] libs: fix the order structcopy documents for ForceSendFields The ForceSend fields come first, then the copied fields the source lists. Say so in the Copy doc comment and in the resource guidelines, which also now mention that enum types are converted. Drop zero values from three struct literals. Co-authored-by: Isaac --- bundle/direct/dresources/README.md | 15 ++++++++------- libs/structs/structcopy/copy.go | 10 +++++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index 4967a112958..243c9815ddc 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -131,13 +131,14 @@ Tell the copier about the exceptions only, each with a comment that says why: - `structcopy.Skip("Owner")` leaves a field unset although the source has it. Use it for a field the request must not carry and for one that is set by hand after the copy (the ID, a value that needs converting). - `structcopy.ForceSend("Comment")` sends a field even when it is zero, so that removing it from the config clears it. Only list fields verified against a real workspace, see `schemaForceSend`. -Do not set `ForceSendFields` next to a copy. The copier derives it: the copied fields that the -source lists, followed by the `ForceSend` ones. The SDK rejects a name its struct does not have, -and the name of a field that was not copied would send that field's zero value. - -A field with the same name but another type or JSON name in the two structs is an error until it -is skipped. `TestFieldCopies` reports it, and records what every copier does with each field of -the struct it builds in `testdata/field_copies.txt`. `./task generate-direct` regenerates that +Do not set `ForceSendFields` next to a copy. The copier derives it: the `ForceSend` fields plus +the copied fields that the source lists. The SDK rejects a name its struct does not have, and the +name of a field that was not copied would send that field's zero value. + +A field with the same name but another JSON name or type in the two structs is an error until it +is skipped; only the enum types the SDK declares once per request and response are converted. +`TestFieldCopies` reports it, and records what every copier does with each field of the struct +it builds in `testdata/field_copies.txt`. `./task generate-direct` regenerates that file. On an SDK bump, read its diff the way you would review the wiring of a new field: it shows which requests and states pick the field up, and which leave it unset because their source has no such field. diff --git a/libs/structs/structcopy/copy.go b/libs/structs/structcopy/copy.go index d204baf1092..bc29f8fe1eb 100644 --- a/libs/structs/structcopy/copy.go +++ b/libs/structs/structcopy/copy.go @@ -70,8 +70,8 @@ func New[Src, Dst any](opts ...Option) *Copier[Src, Dst] { // Copy returns a new Dst holding the fields of src that Dst has as well. The copy is // shallow: maps, slices and pointers are shared with src. // -// ForceSendFields is not copied but derived: it lists the copied fields that src lists, -// followed by the [ForceSend] fields. The SDK marshaller rejects a name its struct lacks +// ForceSendFields is not copied but derived: it lists the [ForceSend] fields, followed by +// the copied fields that src lists. The SDK marshaller rejects a name its struct lacks // and would send the zero value of a field that was left unset. func (c *Copier[Src, Dst]) Copy(src *Src) *Dst { p, err := c.plan() @@ -155,7 +155,7 @@ func newPlan(src, dst reflect.Type, o options) (*plan, error) { } } - p := &plan{src: src, dst: dst, fields: nil, steps: nil, forced: nil} + p := &plan{src: src, dst: dst} for _, name := range o.forceSend { d, ok := dstFields[name] @@ -206,7 +206,7 @@ func newPlan(src, dst reflect.Type, o options) (*plan, error) { } func newCopyStep(s, d field) (copyStep, error) { - step := copyStep{src: s.index, dst: d.index, convertTo: nil} + step := copyStep{src: s.index, dst: d.index} if s.jsonName != d.jsonName { return step, fmt.Errorf("%s is %q in the source's JSON but %q in the destination's; skip it and set it by hand", d.name, s.jsonName, d.jsonName) } @@ -227,7 +227,7 @@ func (p *plan) slot(index []int) *forceSendSlot { return &p.forced[i] } } - p.forced = append(p.forced, forceSendSlot{index: index, always: nil, from: nil}) + p.forced = append(p.forced, forceSendSlot{index: index}) return &p.forced[len(p.forced)-1] } From bacf227ccfe247cb28018a1743309b408c38ee80 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 19 Nov 2026 02:36:21 +0000 Subject: [PATCH 8/9] libs: list the fields structcopy drops, and reject path parameters Describe now lists every field of either struct, so the listing also shows a source field the destination lacks ("not in destination"). For a request built from the state that is a field the request cannot carry, which nothing reported before. A destination field outside the JSON body (json:"-", a path or query parameter) is an error until it is skipped, also when the source field has the same tag. It addresses the resource, so it is never the source's field of the same name. isBasic mirrors the SDK's list of basic kinds, and a test covers ForceSend on a field promoted from an embedded struct. Co-authored-by: Isaac --- .agents/skills/bump-sdk/SKILL.md | 2 +- Taskfile.yml | 7 +- bundle/direct/dresources/README.md | 18 +- .../dresources/testdata/field_copies.txt | 211 ++++++++++++++++++ libs/structs/structcopy/copy.go | 28 ++- libs/structs/structcopy/copy_test.go | 27 +++ 6 files changed, 277 insertions(+), 16 deletions(-) diff --git a/.agents/skills/bump-sdk/SKILL.md b/.agents/skills/bump-sdk/SKILL.md index f87043e63ac..b21d032e571 100644 --- a/.agents/skills/bump-sdk/SKILL.md +++ b/.agents/skills/bump-sdk/SKILL.md @@ -68,7 +68,7 @@ Revert any golden change you can't attribute to the new SDK or spec. **5. Handle SDK breaking changes.** Read the SDK's `CHANGELOG.md` at the target version (in the module cache) to enumerate breaking changes before chasing compile errors. A removed struct field that the CLI used (e.g. `jobs.AiRuntimeTask.CodeSourcePath`) should have its usage temporarily disabled with a comment noting it returns in a later SDK bump, not deleted outright. -A new struct field usually needs no wiring in `bundle/direct/dresources/*`: the structs built with `newCopier` pick it up, and step 4 records that in `bundle/direct/dresources/testdata/field_copies.txt`. Read that file's diff as the review of the wiring: `copied` means a request or state now carries the field, `not in source` means it stays unset because the struct it is copied from has no such field (see "Copying fields between SDK structs" in `bundle/direct/dresources/README.md`). `TestFieldCopies` fails on a field whose type or JSON name differs between the two structs; skip it in the copier and set it by hand. +A new struct field usually needs no wiring in `bundle/direct/dresources/*`: the structs built with `newCopier` pick it up, and step 4 records that in `bundle/direct/dresources/testdata/field_copies.txt`. Read that file's diff as the review of the wiring: `copied` means a request or state now carries the field, `not in source` means it stays unset because the struct it is copied from has no such field, and `not in destination` means the copy drops it (see "Copying fields between SDK structs" in `bundle/direct/dresources/README.md`). A state field that an update request drops cannot be updated in place, so classify it in `.yml`. `TestFieldCopies` fails on a field whose JSON name or type differs between the two structs (types of the same basic kind are converted) and on a path or query parameter; skip it in the copier and set it by hand. A struct literal that is still built by hand triggers an `exhaustruct` lint failure instead. Run `./task lint` and (in case of issues) wire the field through by hand. Where a remote is rebuilt by hand from a GET response (e.g. `makeJobRunRemote` maps `jobs.Run` into the `jobs.RunNow`-shaped `JobRunRemote`), copy the new field from the response if the response has it, and list it with the request-only zero values otherwise. `job_runs.yml` must list every `jobs.RunNow` field under `ignore_remote_changes`; `TestJobRunIgnoresEveryRequestField` fails on a new one. diff --git a/Taskfile.yml b/Taskfile.yml index 8a162dade5f..20d04a6c37b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1102,9 +1102,10 @@ tasks: - "uv run --script bundle/direct/tools/generate_resources.py .codegen/cli.json bundle/direct/dresources/apitypes.generated.yml bundle/direct/dresources/apitypes.yml acceptance/bundle/refschema/output bundle/direct/dresources/configs" # Refreshes the listing of what every structcopy.Copier of the direct engine - # does with each field of the struct it builds (see TestFieldCopies). An SDK - # bump (go.mod / go.sum) that adds a field to such a struct changes it. Runs - # after generate-direct-resources, whose output the package embeds. + # does with each field of the two structs it copies between (see + # TestFieldCopies). An SDK bump (go.mod / go.sum) that adds a field to one of + # them changes it. Runs after generate-direct-resources, whose output the + # package embeds. generate-direct-copies: desc: Regenerate bundle/direct/dresources/testdata/field_copies.txt deps: ['generate-direct-resources'] diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index 243c9815ddc..b1848e9def7 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -136,12 +136,18 @@ the copied fields that the source lists. The SDK rejects a name its struct does name of a field that was not copied would send that field's zero value. A field with the same name but another JSON name or type in the two structs is an error until it -is skipped; only the enum types the SDK declares once per request and response are converted. -`TestFieldCopies` reports it, and records what every copier does with each field of the struct -it builds in `testdata/field_copies.txt`. `./task generate-direct` regenerates that -file. On an SDK bump, read its diff the way you would review the wiring of a new field: it shows -which requests and states pick the field up, and which leave it unset because their source has -no such field. +is skipped. Only a type of the same basic kind is converted, as with the enum types the SDK +declares once per request and response. A destination field outside the JSON body (`json:"-"`, a +path or query parameter) is an error until it is skipped as well: it addresses the resource, so +set it from the ID. + +`TestFieldCopies` reports these errors, and records what every copier does with each field of +either struct in `testdata/field_copies.txt`. `./task generate-direct` regenerates that file. On +an SDK bump, read its diff the way you would review the wiring of a new field: + + - `copied`: the request or state picks the field up. + - `not in source`: the field stays unset, because the struct it is copied from has no such field. + - `not in destination`: the copy drops the field. A state field that an update request drops cannot be updated in place, so classify it in `.yml`. ## OverrideChangeDesc diff --git a/bundle/direct/dresources/testdata/field_copies.txt b/bundle/direct/dresources/testdata/field_copies.txt index a2884048cf8..1d81a1270a6 100644 --- a/bundle/direct/dresources/testdata/field_copies.txt +++ b/bundle/direct/dresources/testdata/field_copies.txt @@ -1,15 +1,29 @@ catalog.CatalogInfo -> dresources.CatalogState + BrowseOnly: not in destination + CatalogType: not in destination Comment: copied ConnectionName: copied + CreatedAt: not in destination + CreatedBy: not in destination CustomMaxRetentionHours: copied + EffectivePredictiveOptimizationFlag: not in destination + EnablePredictiveOptimization: not in destination + FullName: not in destination IsolationMode: copied ManagedEncryptionSettings: copied + MetastoreId: not in destination Name: copied Options: copied + Owner: not in destination Properties: copied ProviderName: copied + ProvisioningInfo: not in destination + SecurableType: not in destination ShareName: copied + StorageLocation: not in destination StorageRoot: copied + UpdatedAt: not in destination + UpdatedBy: not in destination catalog.CreateExternalLocation -> catalog.UpdateExternalLocation Comment: copied, always sent @@ -47,16 +61,23 @@ catalog.CreateRegisteredModelRequest -> catalog.UpdateRegisteredModelRequest UpdatedBy: copied catalog.CreateSchema -> catalog.UpdateSchema + CatalogName: not in destination Comment: copied, always sent CustomMaxRetentionHours: copied, always sent EnablePredictiveOptimization: not in source FullName: not in source + Name: not in destination NewName: not in source Owner: not in source Properties: copied + StorageRoot: not in destination catalog.ExternalLocationInfo -> catalog.CreateExternalLocation + BrowseOnly: not in destination Comment: copied + CreatedAt: not in destination + CreatedBy: not in destination + CredentialId: not in destination CredentialName: copied EffectiveEnableFileEvents: copied EffectiveFileEventQueue: copied @@ -64,24 +85,34 @@ catalog.ExternalLocationInfo -> catalog.CreateExternalLocation EncryptionDetails: copied Fallback: copied FileEventQueue: copied + IsolationMode: not in destination + MetastoreId: not in destination Name: copied + Owner: not in destination ReadOnly: copied SkipValidation: not in source + UpdatedAt: not in destination + UpdatedBy: not in destination Url: copied catalog.MonitorInfo -> dresources.QualityMonitorState AssetsDir: copied BaselineTableName: copied CustomMetrics: copied + DashboardId: not in destination DataClassificationConfig: copied + DriftMetricsTableName: not in destination InferenceLog: copied LatestMonitorFailureMsg: copied + MonitorVersion: not in destination Notifications: copied OutputSchemaName: copied + ProfileMetricsTableName: not in destination Schedule: copied SkipBuiltinDashboard: not in source SlicingExprs: copied Snapshot: copied + Status: not in destination TableName: copied TimeSeries: copied WarehouseId: not in source @@ -103,13 +134,25 @@ catalog.RegisteredModelInfo -> catalog.CreateRegisteredModelRequest UpdatedBy: copied catalog.SchemaInfo -> dresources.SchemaState + BrowseOnly: not in destination CatalogName: copied + CatalogType: not in destination Comment: copied + CreatedAt: not in destination + CreatedBy: not in destination CustomMaxRetentionHours: copied + EffectivePredictiveOptimizationFlag: not in destination + EnablePredictiveOptimization: not in destination + FullName: not in destination + MetastoreId: not in destination Name: copied + Owner: not in destination Properties: copied SchemaId: copied + StorageLocation: not in destination StorageRoot: copied + UpdatedAt: not in destination + UpdatedBy: not in destination catalog.Secret -> catalog.Secret CatalogName: copied @@ -129,11 +172,21 @@ catalog.Secret -> catalog.Secret Value: copied catalog.VolumeInfo -> dresources.VolumeState + AccessPoint: not in destination + BrowseOnly: not in destination CatalogName: copied Comment: copied + CreatedAt: not in destination + CreatedBy: not in destination + EncryptionDetails: not in destination + FullName: not in destination + MetastoreId: not in destination Name: copied + Owner: not in destination SchemaName: copied StorageLocation: copied + UpdatedAt: not in destination + UpdatedBy: not in destination VolumeId: copied VolumeType: copied @@ -214,13 +267,21 @@ compute.ClusterSpec -> compute.EditCluster WorkloadType: copied compute.CreateInstancePool -> compute.EditInstancePool + AwsAttributes: not in destination + AzureAttributes: not in destination CustomTags: copied + DiskSpec: not in destination + EnableElasticDisk: not in destination + GcpAttributes: not in destination IdleInstanceAutoterminationMinutes: copied InstancePoolId: not in source InstancePoolName: copied MaxCapacity: copied MinIdleInstances: copied + NodeTypeFlexibility: not in destination NodeTypeId: copied + PreloadedDockerImages: not in destination + PreloadedSparkVersions: not in destination RemoteDiskThroughput: copied TotalInitialRemoteDiskSize: copied @@ -238,10 +299,12 @@ compute.GetInstancePool -> compute.CreateInstancePool AwsAttributes: copied AzureAttributes: copied CustomTags: copied + DefaultTags: not in destination DiskSpec: copied EnableElasticDisk: copied GcpAttributes: copied IdleInstanceAutoterminationMinutes: copied + InstancePoolId: not in destination InstancePoolName: copied MaxCapacity: copied MinIdleInstances: copied @@ -250,16 +313,23 @@ compute.GetInstancePool -> compute.CreateInstancePool PreloadedDockerImages: copied PreloadedSparkVersions: copied RemoteDiskThroughput: copied + State: not in destination + Stats: not in destination + Status: not in destination TotalInitialRemoteDiskSize: copied compute.Policy -> compute.CreatePolicy + CreatedAtTimestamp: not in destination + CreatorUserName: not in destination Definition: copied Description: copied + IsDefault: not in destination Libraries: copied MaxClustersPerUser: copied Name: copied PolicyFamilyDefinitionOverrides: copied PolicyFamilyId: copied + PolicyId: not in destination dashboards.Dashboard -> dresources.DashboardState CreateTime: copied @@ -278,15 +348,19 @@ dashboards.Dashboard -> dresources.DashboardState WarehouseId: copied dashboards.GenieSpace -> resources.GenieSpaceConfig + CreateTime: not in destination Description: copied Etag: copied ParentPath: copied SerializedSpace: skipped + SpaceId: not in destination Title: copied + UpdateTime: not in destination WarehouseId: copied dresources.CatalogState -> catalog.UpdateCatalog Comment: copied, always sent + ConnectionName: not in destination CustomMaxRetentionHours: copied, always sent EnablePredictiveOptimization: not in source IsolationMode: copied @@ -296,6 +370,9 @@ dresources.CatalogState -> catalog.UpdateCatalog Options: copied Owner: not in source Properties: copied + ProviderName: not in destination + ShareName: not in destination + StorageRoot: not in destination dresources.ClusterRemote -> dresources.ClusterState ApplyPolicyDefaultValues: copied @@ -303,22 +380,34 @@ dresources.ClusterRemote -> dresources.ClusterState AutoterminationMinutes: copied AwsAttributes: copied AzureAttributes: copied + ClusterCores: not in destination + ClusterId: not in destination ClusterLogConf: copied + ClusterLogStatus: not in destination + ClusterMemoryMb: not in destination ClusterName: copied + ClusterSource: not in destination + CreatorUserName: not in destination CustomTags: copied DataSecurityMode: copied + DefaultTags: not in destination DependencyMode: copied DockerImage: copied + Driver: not in destination DriverInstancePoolId: copied DriverNodeTypeFlexibility: copied DriverNodeTypeId: copied EnableElasticDisk: copied EnableLocalDiskEncryption: copied + Executors: not in destination GcpAttributes: copied InitScripts: copied InstancePoolId: copied IsSingleNode: copied + JdbcPort: not in destination Kind: copied + LastRestartedTime: not in destination + LastStateLossTime: not in destination Libraries: copied Lifecycle: copied NodeTypeId: copied @@ -328,40 +417,104 @@ dresources.ClusterRemote -> dresources.ClusterState RuntimeEngine: copied SingleUserName: copied SparkConf: copied + SparkContextId: not in destination SparkEnvVars: copied SparkVersion: copied + Spec: not in destination SshPublicKeys: copied + StartTime: not in destination + State: not in destination + StateMessage: not in destination + TerminatedTime: not in destination + TerminationReason: not in destination TotalInitialRemoteDiskSize: copied UseMlRuntime: copied WorkerNodeTypeFlexibility: copied WorkloadType: copied dresources.ClusterState -> compute.ResizeCluster + ApplyPolicyDefaultValues: not in destination Autoscale: copied + AutoterminationMinutes: not in destination + AwsAttributes: not in destination + AzureAttributes: not in destination ClusterId: not in source + ClusterLogConf: not in destination + ClusterName: not in destination + CustomTags: not in destination + DataSecurityMode: not in destination + DependencyMode: not in destination + DockerImage: not in destination + DriverInstancePoolId: not in destination + DriverNodeTypeFlexibility: not in destination + DriverNodeTypeId: not in destination + EnableElasticDisk: not in destination + EnableLocalDiskEncryption: not in destination + GcpAttributes: not in destination + InitScripts: not in destination + InstancePoolId: not in destination + IsSingleNode: not in destination + Kind: not in destination + Libraries: not in destination + Lifecycle: not in destination + NodeTypeId: not in destination NumWorkers: copied + PolicyId: not in destination + RemoteDiskThroughput: not in destination + RuntimeEngine: not in destination + SingleUserName: not in destination + SparkConf: not in destination + SparkEnvVars: not in destination + SparkVersion: not in destination + SshPublicKeys: not in destination + TotalInitialRemoteDiskSize: not in destination + UseMlRuntime: not in destination + WorkerNodeTypeFlexibility: not in destination + WorkloadType: not in destination dresources.DashboardState -> dashboards.Dashboard CreateTime: skipped DashboardId: skipped + DatasetCatalog: not in destination + DatasetSchema: not in destination DisplayName: copied + EmbedCredentials: not in destination Etag: copied LifecycleState: skipped ParentPath: copied Path: skipped + Published: not in destination SerializedDashboard: skipped UpdateTime: skipped WarehouseId: copied dresources.DashboardState -> dashboards.PublishRequest + CreateTime: not in destination DashboardId: skipped + DatasetCatalog: not in destination + DatasetSchema: not in destination + DisplayName: not in destination EmbedCredentials: copied, always sent + Etag: not in destination + LifecycleState: not in destination + ParentPath: not in destination + Path: not in destination + Published: not in destination + SerializedDashboard: not in destination + UpdateTime: not in destination WarehouseId: copied dresources.MlflowModelRemote -> ml.CreateModelRequest + CreationTimestamp: not in destination Description: copied + Id: not in destination + LastUpdatedTimestamp: not in destination + LatestVersions: not in destination + ModelId: not in destination Name: copied + PermissionLevel: not in destination Tags: copied + UserId: not in destination dresources.ModelServingEndpointRemote -> serving.CreateServingEndpoint AiGateway: copied @@ -369,6 +522,8 @@ dresources.ModelServingEndpointRemote -> serving.CreateServingEndpoint Config: copied Description: copied EmailNotifications: copied + EndpointDetails: not in destination + EndpointId: not in destination Name: copied RateLimits: not in source RouteOptimized: copied @@ -378,6 +533,7 @@ dresources.ModelServingEndpointRemote -> serving.CreateServingEndpoint dresources.PipelineState -> pipelines.EditPipeline AllowDuplicateNames: copied BudgetPolicyId: copied + CascadeOnDestroy: not in destination Catalog: copied Channel: copied Clusters: copied @@ -385,6 +541,7 @@ dresources.PipelineState -> pipelines.EditPipeline Continuous: copied Deployment: copied Development: copied + DryRun: not in destination Edition: copied Environment: copied EventLog: copied @@ -414,6 +571,7 @@ dresources.PipelineState -> pipelines.EditPipeline UsagePolicyId: copied dresources.QualityMonitorState -> catalog.UpdateMonitor + AssetsDir: not in destination BaselineTableName: copied CustomMetrics: copied DashboardId: not in source @@ -423,14 +581,17 @@ dresources.QualityMonitorState -> catalog.UpdateMonitor Notifications: copied OutputSchemaName: copied Schedule: copied + SkipBuiltinDashboard: not in destination SlicingExprs: copied Snapshot: copied TableName: skipped TimeSeries: copied + WarehouseId: not in destination dresources.SecretScopeRemote -> dresources.SecretScopeConfig BackendAzureKeyvault: copied InitialManagePrincipal: not in source + Name: not in destination Scope: copied ScopeBackendType: copied @@ -442,14 +603,21 @@ dresources.SqlWarehouseRemote -> dresources.SqlWarehouseState CustomerVersionLabel: copied EnablePhoton: copied EnableServerlessCompute: copied + Health: not in destination + Id: not in destination InstanceProfileArn: copied + JdbcUrl: not in destination Lifecycle: copied MaxDbu: copied MaxNumClusters: copied MinNumClusters: copied Name: copied + NumActiveSessions: not in destination + NumClusters: not in destination + OdbcParams: not in destination ReydenClusterSizeMultiplier: copied SpotInstancePolicy: copied + State: not in destination StatementTimeout: copied Tags: copied WarehouseType: copied, converted from sql.GetWarehouseResponseWarehouseType @@ -464,6 +632,7 @@ dresources.SqlWarehouseState -> sql.EditWarehouseRequest EnableServerlessCompute: copied Id: not in source InstanceProfileArn: copied + Lifecycle: not in destination MaxDbu: copied MaxNumClusters: copied MinNumClusters: copied @@ -476,26 +645,45 @@ dresources.SqlWarehouseState -> sql.EditWarehouseRequest dresources.VectorSearchEndpointRemote -> vectorsearch.CreateEndpoint BudgetPolicyId: copied + CreationTimestamp: not in destination + Creator: not in destination + CustomTags: not in destination + EffectiveBudgetPolicyId: not in destination + EndpointStatus: not in destination EndpointType: copied + EndpointUuid: not in destination + Id: not in destination + LastUpdatedTimestamp: not in destination + LastUpdatedUser: not in destination Name: copied + NumIndexes: not in destination + ScalingInfo: not in destination TargetQps: copied UsagePolicyId: not in source dresources.VectorSearchIndexRemote -> dresources.VectorSearchIndexState + Creator: not in destination DeltaSyncIndexSpec: skipped DirectAccessIndexSpec: copied + EndpointId: not in destination EndpointName: copied EndpointUuid: copied IndexSubtype: copied IndexType: copied Name: copied PrimaryKey: copied + Status: not in destination dresources.VolumeState -> catalog.UpdateVolumeRequestContent + CatalogName: not in destination Comment: copied, always sent Name: skipped NewName: not in source Owner: not in source + SchemaName: not in destination + StorageLocation: not in destination + VolumeId: not in destination + VolumeType: not in destination jobs.JobSettings -> jobs.CreateJob AccessControlList: not in source @@ -532,9 +720,14 @@ jobs.JobSettings -> jobs.CreateJob ml.CreateModelRequest -> ml.UpdateModelRequest Description: copied Name: skipped + Tags: not in destination ml.Experiment -> ml.CreateExperiment ArtifactLocation: copied + CreationTime: not in destination + ExperimentId: not in destination + LastUpdateTime: not in destination + LifecycleStage: not in destination Name: copied Tags: copied TraceLocation: copied @@ -608,6 +801,7 @@ pipelines.PipelineSpec -> pipelines.CreatePipeline resources.GenieSpaceConfig -> dashboards.GenieCreateSpaceRequest Description: copied + Etag: not in destination ParentPath: copied SerializedSpace: skipped Title: copied @@ -634,10 +828,12 @@ serving.AutoCaptureConfigOutput -> serving.AutoCaptureConfigInput CatalogName: copied Enabled: copied SchemaName: copied + State: not in destination TableNamePrefix: copied serving.EndpointCoreConfigOutput -> serving.EndpointCoreConfigInput AutoCaptureConfig: skipped + ConfigVersion: not in destination Name: not in source ServedEntities: skipped ServedModels: skipped @@ -645,10 +841,13 @@ serving.EndpointCoreConfigOutput -> serving.EndpointCoreConfigInput serving.ServedEntityOutput -> serving.ServedEntityInput BurstScalingEnabled: copied + CreationTimestamp: not in destination + Creator: not in destination EntityName: copied EntityVersion: copied EnvironmentVars: copied ExternalModel: copied + FoundationModel: not in destination InstanceProfileArn: copied MaxProvisionedConcurrency: copied MaxProvisionedThroughput: copied @@ -657,6 +856,7 @@ serving.ServedEntityOutput -> serving.ServedEntityInput Name: copied ProvisionedModelUnits: copied ScaleToZeroEnabled: copied + State: not in destination WorkloadSize: copied WorkloadType: copied @@ -664,13 +864,23 @@ serving.ServingEndpointDetailed -> dresources.ModelServingEndpointRemote AiGateway: copied BudgetPolicyId: copied Config: skipped + CreationTimestamp: not in destination + Creator: not in destination + DataPlaneInfo: not in destination Description: copied EmailNotifications: copied EndpointDetails: not in source EndpointId: not in source + EndpointUrl: not in destination + Id: not in destination + LastUpdatedTimestamp: not in destination Name: copied + PendingConfig: not in destination + PermissionLevel: not in destination RouteOptimized: copied + State: not in destination Tags: copied + Task: not in destination TelemetryConfig: copied vectorsearch.DeltaSyncVectorIndexSpecResponse -> vectorsearch.DeltaSyncVectorIndexSpecRequest @@ -679,5 +889,6 @@ vectorsearch.DeltaSyncVectorIndexSpecResponse -> vectorsearch.DeltaSyncVectorInd EmbeddingSourceColumns: copied EmbeddingVectorColumns: copied EmbeddingWritebackTable: copied + PipelineId: not in destination PipelineType: copied SourceTable: copied diff --git a/libs/structs/structcopy/copy.go b/libs/structs/structcopy/copy.go index bc29f8fe1eb..7b775642a3e 100644 --- a/libs/structs/structcopy/copy.go +++ b/libs/structs/structcopy/copy.go @@ -83,7 +83,7 @@ func (c *Copier[Src, Dst]) Copy(src *Src) *Dst { return dst } -// Describe lists what the Copier does with every field of the destination, one line per +// Describe lists what the Copier does with every field of either struct, one line per // field, or reports why the Copier is invalid. func (c *Copier[Src, Dst]) Describe() (string, error) { p, err := c.plan() @@ -102,7 +102,7 @@ func (c *Copier[Src, Dst]) Describe() (string, error) { type plan struct { src, dst reflect.Type - // fields describes every destination field, sorted by name. + // fields describes every field of either struct, sorted by name. fields []fieldOutcome steps []copyStep forced []forceSendSlot @@ -172,12 +172,22 @@ func newPlan(src, dst reflect.Type, o options) (*plan, error) { slot.always = append(slot.always, name) } - for _, name := range slices.Sorted(maps.Keys(dstFields)) { - d := dstFields[name] + names := slices.Collect(maps.Keys(dstFields)) + for name := range srcFields { + if _, ok := dstFields[name]; !ok { + names = append(names, name) + } + } + slices.Sort(names) + + for _, name := range names { + d, inDestination := dstFields[name] s, inSource := srcFields[name] var outcome string switch { + case !inDestination: + outcome = "not in destination" case slices.Contains(o.skip, name): outcome = "skipped" case !inSource: @@ -207,6 +217,11 @@ func newPlan(src, dst reflect.Type, o options) (*plan, error) { func newCopyStep(s, d field) (copyStep, error) { step := copyStep{src: s.index, dst: d.index} + if d.jsonName == "-" { + // A path or query parameter addresses the resource, so its value is not the + // caller's field of the same name: UpdateCatalog.Name is the catalog to update. + return step, fmt.Errorf("%s is not in the destination's JSON (json:\"-\"); skip it and set it by hand", d.name) + } if s.jsonName != d.jsonName { return step, fmt.Errorf("%s is %q in the source's JSON but %q in the destination's; skip it and set it by hand", d.name, s.jsonName, d.jsonName) } @@ -328,8 +343,9 @@ func isBasic(kind reflect.Kind) bool { switch kind { case reflect.Bool, reflect.String, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64: + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, + reflect.Complex64, reflect.Complex128: return true default: return false diff --git a/libs/structs/structcopy/copy_test.go b/libs/structs/structcopy/copy_test.go index be4c7ac0116..33dd3456a3e 100644 --- a/libs/structs/structcopy/copy_test.go +++ b/libs/structs/structcopy/copy_test.go @@ -159,6 +159,15 @@ func TestCopyKeepsForceSendFieldsPerStruct(t *testing.T) { assert.Equal(t, []string{"ThingId"}, got.ForceSendFields) } +func TestForceSendOnPromotedFieldUsesForceSendFieldsOfItsStruct(t *testing.T) { + copier := structcopy.New[remoteThing, stateWithOwnForceSend](structcopy.ForceSend("ThingId", "Comment")) + + got := copier.Copy(&remoteThing{}) + + assert.Equal(t, []string{"Comment"}, got.createThing.ForceSendFields) + assert.Equal(t, []string{"ThingId"}, got.ForceSendFields) +} + type withUnexportedField struct { Name string `json:"name,omitempty"` secret string @@ -237,11 +246,14 @@ func TestDescribe(t *testing.T) { require.NoError(t, err) assert.Equal(t, `structcopy_test.thingState -> structcopy_test.updateThing Comment: copied, always sent + Kind: not in destination Name: skipped NewName: not in source Owner: not in source Size: copied + SkipValidation: not in destination Tags: copied + ThingId: not in destination `, got) } @@ -265,6 +277,9 @@ type ( embeddedPointer struct { *createThing } + getThing struct { + Name string `json:"-" url:"-"` + } ) func TestInvalidCopier(t *testing.T) { @@ -298,6 +313,18 @@ func TestInvalidCopier(t *testing.T) { copy: func() { structcopy.New[createThing, sizeRenamedOnWire]().Copy(&createThing{}) }, want: `structcopy: structcopy_test.createThing -> structcopy_test.sizeRenamedOnWire: Size is "size" in the source's JSON but "capacity" in the destination's; skip it and set it by hand`, }, + { + name: "field is a path parameter in the destination", + describe: structcopy.New[createThing, updateThing]().Describe, + copy: func() { structcopy.New[createThing, updateThing]().Copy(&createThing{}) }, + want: `structcopy: structcopy_test.createThing -> structcopy_test.updateThing: Name is not in the destination's JSON (json:"-"); skip it and set it by hand`, + }, + { + name: "field is a path parameter in both", + describe: structcopy.New[getThing, updateThing]().Describe, + copy: func() { structcopy.New[getThing, updateThing]().Copy(&getThing{}) }, + want: `structcopy: structcopy_test.getThing -> structcopy_test.updateThing: Name is not in the destination's JSON (json:"-"); skip it and set it by hand`, + }, { name: "skipped field is not in the destination", describe: structcopy.New[createThing, updateThing](structcopy.Skip("Name", "Kind")).Describe, From 592ca0d61f9dc21d6faf0ce7908fda333314c5e0 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 19 Nov 2026 02:36:40 +0000 Subject: [PATCH 9/9] direct: restore notes on deprecated pipeline fields and unused job settings The nolint comments said why the deprecated continuous and development fields of a pipeline are still mapped, and makeCreateJob noted that nothing checks whether every job setting is used. Carry both over to the copiers. Name the dashboard's force-sent field once. Co-authored-by: Isaac --- bundle/direct/dresources/cluster.go | 5 +++-- bundle/direct/dresources/dashboard.go | 14 +++++++------- bundle/direct/dresources/job.go | 3 ++- bundle/direct/dresources/pipeline.go | 2 ++ 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index d54d70f3b31..b5abb6b4453 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -385,9 +385,10 @@ func remoteClusterIsRunning(entry *PlanEntry) bool { return remote.State == compute.StateRunning } +// CreateCluster.CloneFrom stays empty: it is not supported by DABs. +var toCreateCluster = newCopier[compute.ClusterSpec, compute.CreateCluster]() + var ( - // CloneFrom is not supported by DABs. - toCreateCluster = newCopier[compute.ClusterSpec, compute.CreateCluster]() toEditCluster = newCopier[compute.ClusterSpec, compute.EditCluster]() toResizeCluster = newCopier[ClusterState, compute.ResizeCluster]() ) diff --git a/bundle/direct/dresources/dashboard.go b/bundle/direct/dresources/dashboard.go index ff68804cbe6..61560799d73 100644 --- a/bundle/direct/dresources/dashboard.go +++ b/bundle/direct/dresources/dashboard.go @@ -143,6 +143,11 @@ func publishedIsCurrent(dashboard *dashboards.Dashboard, published *dashboards.P return !revisionTime.Before(updateTime) } +// embed_credentials as a zero valued default in resourcemutator/resource_mutator.go. +// Thus we always need to include it in the ForceSendFields list to ensure that it is sent to +// the server, and that it's serialized in the state even when false (zero value). +var dashboardForceSend = []string{"EmbedCredentials"} + // serialized_dashboard is a string in the API and any in the config, so each copier skips it // and the caller sets it. var ( @@ -152,23 +157,18 @@ var ( structcopy.Skip("CreateTime", "DashboardId", "LifecycleState", "Path", "UpdateTime", "SerializedDashboard"), ) - // embed_credentials as a zero valued default in resourcemutator/resource_mutator.go. - // Thus we always need to include it in the ForceSendFields list to ensure that it is sent to the server. // DashboardId addresses the dashboard, so it is the id. toPublishRequest = newCopier[DashboardState, dashboards.PublishRequest]( structcopy.Skip("DashboardId"), - structcopy.ForceSend("EmbedCredentials"), + structcopy.ForceSend(dashboardForceSend...), ) // The output-only fields (create_time, path, ...) are copied as well. dataset_catalog and // dataset_schema are not reported: diffs are detected via etags, which will change if // dataset_catalog/dataset_schema is updated. - // - // EmbedCredentials must always be included in ForceSendFields to ensure it's serialized - // even when false (zero value). toDashboardState = newCopier[dashboards.Dashboard, DashboardState]( structcopy.Skip("SerializedDashboard"), - structcopy.ForceSend("EmbedCredentials"), + structcopy.ForceSend(dashboardForceSend...), ) ) diff --git a/bundle/direct/dresources/job.go b/bundle/direct/dresources/job.go index 017552a00c2..b3bb1e9dbcb 100644 --- a/bundle/direct/dresources/job.go +++ b/bundle/direct/dresources/job.go @@ -177,7 +177,8 @@ func (r *ResourceJob) DoDelete(ctx context.Context, id string, _ *jobs.JobSettin return r.client.Jobs.DeleteByJobId(ctx, idInt) } -// AccessControlList stays empty: it is not supported by DABs. +// AccessControlList stays empty: it is not supported by DABs. A JobSettings field that +// CreateJob lacks would be dropped; testdata/field_copies.txt lists it as "not in destination". var toCreateJob = newCopier[jobs.JobSettings, jobs.CreateJob]() func makeCreateJob(config jobs.JobSettings) (jobs.CreateJob, error) { diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 40736048b6a..805631dbf09 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -97,6 +97,8 @@ func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemo } // AllowDuplicateNames and DryRun are request-only, so PipelineSpec has no value for them. +// Continuous and Development are deprecated in the SDK but remain supported bundle config +// fields, so they are copied here and into the edit request like any other field. var toCreatePipeline = newCopier[pipelines.PipelineSpec, pipelines.CreatePipeline]() func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote {