diff --git a/.nextchanges/bundles/deployment-history-state-size.md b/.nextchanges/bundles/deployment-history-state-size.md new file mode 100644 index 00000000000..6f79d21fea7 --- /dev/null +++ b/.nextchanges/bundles/deployment-history-state-size.md @@ -0,0 +1 @@ +* Raise the uncompressed deployment-history state limit to 800 KiB while retaining the server's 64 KiB compressed storage limit. ([#7840](https://github.com/databricks/cli/pull/7840)) diff --git a/libs/dms/operation.go b/libs/dms/operation.go index 3bb2f0b00d8..14ed295ace9 100644 --- a/libs/dms/operation.go +++ b/libs/dms/operation.go @@ -13,10 +13,9 @@ import ( // ("resources.jobs.foo" vs "jobs.foo"). Strip it when sending, add it back when reading. const StatePrefix = "resources." -// maxStateSize is the largest serialized state DMS accepts per operation. More than this -// and the resource cannot be recorded at all, so the deploy fails rather than leaving the -// service holding a resource with no state. -const maxStateSize = 64 * 1024 +// maxStateSize caps the uncompressed state sent to DMS. The service compresses +// the state and enforces a separate 64 KiB limit on the compressed bytes. +const maxStateSize = 800 * 1024 // maxErrorMessageSize is how much of a failure's message the service stores. const maxErrorMessageSize = 16 * 1024 diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go index 6aa6aea4943..73eedb79d5f 100644 --- a/libs/dms/operation_buffer_test.go +++ b/libs/dms/operation_buffer_test.go @@ -64,15 +64,14 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { } func TestBufferFailsOnOversizedState(t *testing.T) { - // The service will not take a state this large, so the resource cannot be recorded. - // Failing here says so, where reporting nothing would leave DMS without the resource - // and the next plan would create it again. + // State over the client limit is rejected before upload and counted as a recording + // failure rather than left pending. s := queued() s.stats.StagedOperationCount = 1 s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", json.RawMessage(strings.Repeat("x", maxStateSize+1))) - assert.ErrorContains(t, s.Err(), "exceeds the 65536 byte limit") + assert.ErrorContains(t, s.Err(), "exceeds the 819200 byte limit") assert.Empty(t, s.queue) assert.Empty(t, s.pending) assert.Equal(t, 1, s.Stats().OperationRecordingFailureCount) diff --git a/libs/dms/operation_test.go b/libs/dms/operation_test.go index 2bae9a81144..ccae02c18da 100644 --- a/libs/dms/operation_test.go +++ b/libs/dms/operation_test.go @@ -16,6 +16,31 @@ import ( // failure leaves alone - is asserted by acceptance/bundle/dms. What is left here are the limits // and the merge, which a deploy cannot reach. +func TestNewStateUpdateSizeLimit(t *testing.T) { + for _, tc := range []struct { + name string + size int + wantErr bool + }{ + {name: "above compressed storage limit", size: 80 * 1024}, + {name: "below client limit", size: 800*1024 - 1}, + {name: "at client limit", size: 800 * 1024}, + {name: "above client limit", size: 800*1024 + 1, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + state := json.RawMessage(`{"state":"` + strings.Repeat("x", tc.size-len(`{"state":""}`)) + `"}`) + update, err := NewStateUpdate("job-123", state, false) + if tc.wantErr { + require.ErrorContains(t, err, "exceeds the 819200 byte limit") + return + } + require.NoError(t, err) + assert.Len(t, update.State, tc.size) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, update.Status) + }) + } +} + func TestNewFailureUpdateTruncatesLongError(t *testing.T) { // Truncated rather than rejected: a message over the limit would make recording // fail and hide the error it is reporting.