Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/deployment-history-state-size.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Raise the uncompressed deployment-history state limit to 800 KiB while retaining the server's 64 KiB compressed storage limit. ([#7840](https://github.com/databricks/cli/pull/7840))
7 changes: 3 additions & 4 deletions libs/dms/operation.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,9 @@ import (
// ("resources.jobs.foo" vs "jobs.foo"). Strip it when sending, add it back when reading.
const StatePrefix = "resources."

// maxStateSize is the largest serialized state DMS accepts per operation. More than this
// and the resource cannot be recorded at all, so the deploy fails rather than leaving the
// service holding a resource with no state.
const maxStateSize = 64 * 1024
// maxStateSize caps the uncompressed state sent to DMS. The service compresses
// the state and enforces a separate 64 KiB limit on the compressed bytes.
const maxStateSize = 800 * 1024

// maxErrorMessageSize is how much of a failure's message the service stores.
const maxErrorMessageSize = 16 * 1024
Expand Down
7 changes: 3 additions & 4 deletions libs/dms/operation_buffer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,15 +64,14 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) {
}

func TestBufferFailsOnOversizedState(t *testing.T) {
// The service will not take a state this large, so the resource cannot be recorded.
// Failing here says so, where reporting nothing would leave DMS without the resource
// and the next plan would create it again.
// State over the client limit is rejected before upload and counted as a recording
// failure rather than left pending.
s := queued()
s.stats.StagedOperationCount = 1

s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", json.RawMessage(strings.Repeat("x", maxStateSize+1)))

assert.ErrorContains(t, s.Err(), "exceeds the 65536 byte limit")
assert.ErrorContains(t, s.Err(), "exceeds the 819200 byte limit")
assert.Empty(t, s.queue)
assert.Empty(t, s.pending)
assert.Equal(t, 1, s.Stats().OperationRecordingFailureCount)
Expand Down
25 changes: 25 additions & 0 deletions libs/dms/operation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,31 @@ import (
// failure leaves alone - is asserted by acceptance/bundle/dms. What is left here are the limits
// and the merge, which a deploy cannot reach.

func TestNewStateUpdateSizeLimit(t *testing.T) {
for _, tc := range []struct {
name string
size int
wantErr bool
}{
{name: "above compressed storage limit", size: 80 * 1024},
{name: "below client limit", size: 800*1024 - 1},
{name: "at client limit", size: 800 * 1024},
{name: "above client limit", size: 800*1024 + 1, wantErr: true},
} {
t.Run(tc.name, func(t *testing.T) {
state := json.RawMessage(`{"state":"` + strings.Repeat("x", tc.size-len(`{"state":""}`)) + `"}`)
update, err := NewStateUpdate("job-123", state, false)
if tc.wantErr {
require.ErrorContains(t, err, "exceeds the 819200 byte limit")
return
}
require.NoError(t, err)
assert.Len(t, update.State, tc.size)
assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, update.Status)
})
}
}

func TestNewFailureUpdateTruncatesLongError(t *testing.T) {
// Truncated rather than rejected: a message over the limit would make recording
// fail and hide the error it is reporting.
Expand Down
Loading