From 63ee801910fabb1008c68ab5ebc812323b00ff18 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 20 Nov 2026 08:36:18 +0000 Subject: [PATCH 1/3] bundle: match deployment history uncompressed state limit --- .../bundles/deployment-history-state-size.md | 1 + libs/dms/operation.go | 7 +++--- libs/dms/operation_buffer_test.go | 2 +- libs/dms/operation_test.go | 24 +++++++++++++++++++ 4 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 .nextchanges/bundles/deployment-history-state-size.md diff --git a/.nextchanges/bundles/deployment-history-state-size.md b/.nextchanges/bundles/deployment-history-state-size.md new file mode 100644 index 00000000000..02c0db436f7 --- /dev/null +++ b/.nextchanges/bundles/deployment-history-state-size.md @@ -0,0 +1 @@ +* Fix deployment history recording for large resource states that compress below the server's 64 KiB storage limit. diff --git a/libs/dms/operation.go b/libs/dms/operation.go index 3bb2f0b00d8..f30e12d0082 100644 --- a/libs/dms/operation.go +++ b/libs/dms/operation.go @@ -13,10 +13,9 @@ import ( // ("resources.jobs.foo" vs "jobs.foo"). Strip it when sending, add it back when reading. const StatePrefix = "resources." -// maxStateSize is the largest serialized state DMS accepts per operation. More than this -// and the resource cannot be recorded at all, so the deploy fails rather than leaving the -// service holding a resource with no state. -const maxStateSize = 64 * 1024 +// maxStateSize matches DMS's 10 MiB uncompressed safety limit. The service compresses +// the state and enforces a separate 64 KiB limit on the compressed bytes. +const maxStateSize = 10 * 1024 * 1024 // maxErrorMessageSize is how much of a failure's message the service stores. const maxErrorMessageSize = 16 * 1024 diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go index 6aa6aea4943..39a3f8ec5b7 100644 --- a/libs/dms/operation_buffer_test.go +++ b/libs/dms/operation_buffer_test.go @@ -72,7 +72,7 @@ func TestBufferFailsOnOversizedState(t *testing.T) { s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", json.RawMessage(strings.Repeat("x", maxStateSize+1))) - assert.ErrorContains(t, s.Err(), "exceeds the 65536 byte limit") + assert.ErrorContains(t, s.Err(), "exceeds the 10485760 byte limit") assert.Empty(t, s.queue) assert.Empty(t, s.pending) assert.Equal(t, 1, s.Stats().OperationRecordingFailureCount) diff --git a/libs/dms/operation_test.go b/libs/dms/operation_test.go index 2bae9a81144..884fbc2199c 100644 --- a/libs/dms/operation_test.go +++ b/libs/dms/operation_test.go @@ -16,6 +16,30 @@ import ( // failure leaves alone - is asserted by acceptance/bundle/dms. What is left here are the limits // and the merge, which a deploy cannot reach. +func TestNewStateUpdateSizeLimit(t *testing.T) { + for _, tc := range []struct { + name string + size int + wantErr bool + }{ + {name: "above compressed storage limit", size: 80 * 1024}, + {name: "at uncompressed limit", size: 10 * 1024 * 1024}, + {name: "above uncompressed limit", size: 10*1024*1024 + 1, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + state := json.RawMessage(`{"state":"` + strings.Repeat("x", tc.size-len(`{"state":""}`)) + `"}`) + update, err := NewStateUpdate("job-123", state, false) + if tc.wantErr { + require.ErrorContains(t, err, "exceeds the 10485760 byte limit") + return + } + require.NoError(t, err) + assert.Len(t, update.State, tc.size) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, update.Status) + }) + } +} + func TestNewFailureUpdateTruncatesLongError(t *testing.T) { // Truncated rather than rejected: a message over the limit would make recording // fail and hide the error it is reporting. From 07a2825277d594d1e775a549e26329edd1d51c2c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 20 Nov 2026 08:37:34 +0000 Subject: [PATCH 2/3] docs: link deployment history size fix in changelog --- .nextchanges/bundles/deployment-history-state-size.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/deployment-history-state-size.md b/.nextchanges/bundles/deployment-history-state-size.md index 02c0db436f7..959eaacb38d 100644 --- a/.nextchanges/bundles/deployment-history-state-size.md +++ b/.nextchanges/bundles/deployment-history-state-size.md @@ -1 +1 @@ -* Fix deployment history recording for large resource states that compress below the server's 64 KiB storage limit. +* Fix deployment history recording for large resource states that compress below the server's 64 KiB storage limit. ([#7840](https://github.com/databricks/cli/pull/7840)) From b427afc3826d76ef57b45dcce7e79295b0899c79 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 20 Nov 2026 10:35:53 +0000 Subject: [PATCH 3/3] bundle: set deployment history state cap to 800 KiB --- .nextchanges/bundles/deployment-history-state-size.md | 2 +- libs/dms/operation.go | 4 ++-- libs/dms/operation_buffer_test.go | 7 +++---- libs/dms/operation_test.go | 7 ++++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.nextchanges/bundles/deployment-history-state-size.md b/.nextchanges/bundles/deployment-history-state-size.md index 959eaacb38d..6f79d21fea7 100644 --- a/.nextchanges/bundles/deployment-history-state-size.md +++ b/.nextchanges/bundles/deployment-history-state-size.md @@ -1 +1 @@ -* Fix deployment history recording for large resource states that compress below the server's 64 KiB storage limit. ([#7840](https://github.com/databricks/cli/pull/7840)) +* Raise the uncompressed deployment-history state limit to 800 KiB while retaining the server's 64 KiB compressed storage limit. ([#7840](https://github.com/databricks/cli/pull/7840)) diff --git a/libs/dms/operation.go b/libs/dms/operation.go index f30e12d0082..14ed295ace9 100644 --- a/libs/dms/operation.go +++ b/libs/dms/operation.go @@ -13,9 +13,9 @@ import ( // ("resources.jobs.foo" vs "jobs.foo"). Strip it when sending, add it back when reading. const StatePrefix = "resources." -// maxStateSize matches DMS's 10 MiB uncompressed safety limit. The service compresses +// maxStateSize caps the uncompressed state sent to DMS. The service compresses // the state and enforces a separate 64 KiB limit on the compressed bytes. -const maxStateSize = 10 * 1024 * 1024 +const maxStateSize = 800 * 1024 // maxErrorMessageSize is how much of a failure's message the service stores. const maxErrorMessageSize = 16 * 1024 diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go index 39a3f8ec5b7..73eedb79d5f 100644 --- a/libs/dms/operation_buffer_test.go +++ b/libs/dms/operation_buffer_test.go @@ -64,15 +64,14 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { } func TestBufferFailsOnOversizedState(t *testing.T) { - // The service will not take a state this large, so the resource cannot be recorded. - // Failing here says so, where reporting nothing would leave DMS without the resource - // and the next plan would create it again. + // State over the client limit is rejected before upload and counted as a recording + // failure rather than left pending. s := queued() s.stats.StagedOperationCount = 1 s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", json.RawMessage(strings.Repeat("x", maxStateSize+1))) - assert.ErrorContains(t, s.Err(), "exceeds the 10485760 byte limit") + assert.ErrorContains(t, s.Err(), "exceeds the 819200 byte limit") assert.Empty(t, s.queue) assert.Empty(t, s.pending) assert.Equal(t, 1, s.Stats().OperationRecordingFailureCount) diff --git a/libs/dms/operation_test.go b/libs/dms/operation_test.go index 884fbc2199c..ccae02c18da 100644 --- a/libs/dms/operation_test.go +++ b/libs/dms/operation_test.go @@ -23,14 +23,15 @@ func TestNewStateUpdateSizeLimit(t *testing.T) { wantErr bool }{ {name: "above compressed storage limit", size: 80 * 1024}, - {name: "at uncompressed limit", size: 10 * 1024 * 1024}, - {name: "above uncompressed limit", size: 10*1024*1024 + 1, wantErr: true}, + {name: "below client limit", size: 800*1024 - 1}, + {name: "at client limit", size: 800 * 1024}, + {name: "above client limit", size: 800*1024 + 1, wantErr: true}, } { t.Run(tc.name, func(t *testing.T) { state := json.RawMessage(`{"state":"` + strings.Repeat("x", tc.size-len(`{"state":""}`)) + `"}`) update, err := NewStateUpdate("job-123", state, false) if tc.wantErr { - require.ErrorContains(t, err, "exceeds the 10485760 byte limit") + require.ErrorContains(t, err, "exceeds the 819200 byte limit") return } require.NoError(t, err)