From c511f73758452cea2e38af5482c5be025cfeeb81 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 12:47:49 +0000 Subject: [PATCH 1/3] acc: leave requests answered with a transient error out of cloud recordings On a real workspace the CLI repeats a request answered with a 5xx, 408 or 429, and the proxy recorded both attempts, so out.requests goldens failed whenever the workspace had a hiccup. The proxy now removes the failed attempt from out.requests.txt once its response arrives. cloud: TestAccept/selftest/record_cloud TestAccept/bundle/resources/postgres_roles/update TestAccept/bundle/resources/postgres_databases/basic Co-authored-by: Isaac --- acceptance/internal/prepare_server.go | 109 +++++++++++++++--- .../record_proxy_retry/databricks.yml | 12 ++ .../selftest/record_proxy_retry/out.test.toml | 2 + .../selftest/record_proxy_retry/output.txt | 36 ++++++ acceptance/selftest/record_proxy_retry/script | 11 ++ .../selftest/record_proxy_retry/test.toml | 6 + 6 files changed, 159 insertions(+), 17 deletions(-) create mode 100644 acceptance/selftest/record_proxy_retry/databricks.yml create mode 100644 acceptance/selftest/record_proxy_retry/out.test.toml create mode 100644 acceptance/selftest/record_proxy_retry/output.txt create mode 100644 acceptance/selftest/record_proxy_retry/script create mode 100644 acceptance/selftest/record_proxy_retry/test.toml diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 6fca1f6f0ec..d6ace0c3a46 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -220,27 +220,90 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o return cfg, testUser, strconv.Itoa(testserver.TestWorkspaceID) } -func recordRequestsCallback(t *testing.T, includeHeaders []string, outputDir string) func(request *testserver.Request) { - mu := sync.Mutex{} +// requestRecorder appends each request to out.requests.txt in outputDir as it arrives. +type requestRecorder struct { + t *testing.T + includeHeaders []string + outputDir string + + mu sync.Mutex + // entries holds what was written for requests still waiting for a response; nil unless + // dropRetried is used. + entries map[*testserver.Request]string +} - return func(request *testserver.Request) { - mu.Lock() - defer mu.Unlock() +func newRequestRecorder(t *testing.T, includeHeaders []string, outputDir string, dropRetried bool) *requestRecorder { + r := &requestRecorder{ + t: t, + includeHeaders: includeHeaders, + outputDir: outputDir, + } + if dropRetried { + r.entries = map[*testserver.Request]string{} + } + return r +} - req := getLoggedRequest(request, includeHeaders) - reqJson, err := json.MarshalIndent(req, "", " ") - assert.NoErrorf(t, err, "Failed to json-encode: %#v", req) +func (r *requestRecorder) path() string { + return filepath.Join(r.outputDir, "out.requests.txt") +} + +func (r *requestRecorder) record(request *testserver.Request) { + r.mu.Lock() + defer r.mu.Unlock() + + req := getLoggedRequest(request, r.includeHeaders) + reqJson, err := json.MarshalIndent(req, "", " ") + assert.NoErrorf(r.t, err, "Failed to json-encode: %#v", req) + entry := string(reqJson) + "\n" - requestsPath := filepath.Join(outputDir, "out.requests.txt") - f, err := os.OpenFile(requestsPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) - assert.NoError(t, err) - defer f.Close() + f, err := os.OpenFile(r.path(), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) + assert.NoError(r.t, err) + defer f.Close() - _, err = f.WriteString(string(reqJson) + "\n") - assert.NoError(t, err) + _, err = f.WriteString(entry) + assert.NoError(r.t, err) + if r.entries != nil { + r.entries[request] = entry } } +// isRetriedStatus reports whether the CLI retries a request answered with status (see +// libs/retry and the SDK's own retries). +func isRetriedStatus(status int) bool { + return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500 +} + +// dropRetried removes a request answered with a transient error from out.requests.txt. A real +// workspace answers with such errors now and then; the CLI repeats the request and the repeat is +// recorded, so dropping the failed attempt keeps the goldens stable. Only the proxy uses it: the +// testserver answers with an error only when a test injects one, and its goldens show that. +func (r *requestRecorder) dropRetried(request *testserver.Request, response *testserver.EncodedResponse) { + r.mu.Lock() + defer r.mu.Unlock() + + entry, ok := r.entries[request] + delete(r.entries, request) + if !ok || !isRetriedStatus(response.StatusCode) { + return + } + + data, err := os.ReadFile(r.path()) + if errors.Is(err, os.ErrNotExist) { + // The script removed the file after the request was sent. + return + } + require.NoError(r.t, err) + + i := strings.LastIndex(string(data), entry) + if i < 0 { + return + } + data = append(data[:i:i], data[i+len(entry):]...) + require.NoError(r.t, os.WriteFile(r.path(), data, 0o644)) + r.t.Logf("Dropped from out.requests.txt after %d response: %s %s", response.StatusCode, request.Method, request.URL.Path) +} + func logResponseCallback(t *testing.T) func(request *testserver.Request, response *testserver.EncodedResponse) { mu := sync.Mutex{} @@ -268,7 +331,7 @@ func startLocalServer(t *testing.T, // Record API requests in out.requests.txt if RecordRequests is true // in test.toml if recordRequests { - s.RequestCallback = recordRequestsCallback(t, includeHeaders, outputDir) + s.RequestCallback = newRequestRecorder(t, includeHeaders, outputDir, false).record } // Log API responses if the -logrequests flag is set. @@ -322,14 +385,26 @@ func startProxyServer(t *testing.T, ) string { s := testproxy.New(t, upstream) + var responseCallbacks []func(request *testserver.Request, response *testserver.EncodedResponse) + // Record API requests in out.requests.txt if RecordRequests is true in test.toml. if recordRequests { - s.RequestCallback = recordRequestsCallback(t, includeHeaders, outputDir) + recorder := newRequestRecorder(t, includeHeaders, outputDir, true) + s.RequestCallback = recorder.record + responseCallbacks = append(responseCallbacks, recorder.dropRetried) } // Log API responses if the -logrequests flag is set. if logRequests { - s.ResponseCallback = logResponseCallback(t) + responseCallbacks = append(responseCallbacks, logResponseCallback(t)) + } + + if len(responseCallbacks) > 0 { + s.ResponseCallback = func(request *testserver.Request, response *testserver.EncodedResponse) { + for _, callback := range responseCallbacks { + callback(request, response) + } + } } return s.URL diff --git a/acceptance/selftest/record_proxy_retry/databricks.yml b/acceptance/selftest/record_proxy_retry/databricks.yml new file mode 100644 index 00000000000..b95ebe35d2c --- /dev/null +++ b/acceptance/selftest/record_proxy_retry/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Postgres Project" + pg_version: 16 diff --git a/acceptance/selftest/record_proxy_retry/out.test.toml b/acceptance/selftest/record_proxy_retry/out.test.toml new file mode 100644 index 00000000000..15b82625790 --- /dev/null +++ b/acceptance/selftest/record_proxy_retry/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.READPLAN = [] diff --git a/acceptance/selftest/record_proxy_retry/output.txt b/acceptance/selftest/record_proxy_retry/output.txt new file mode 100644 index 00000000000..74b36996c68 --- /dev/null +++ b/acceptance/selftest/record_proxy_retry/output.txt @@ -0,0 +1,36 @@ + +=== A request answered with a transient error is left out of the recorded requests; the CLI's repeat is recorded + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj" + }, + "body": { + "spec": { + "display_name": "Test Postgres Project", + "pg_version": 16 + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj/operations/[UUID]" +} + +=== A request still failing when the CLI gives up is left out as well; the output shows the error + +>>> [CLI] current-user me +Error: Fault injected by test. + +Exit code: 1 + +>>> print_requests.py --get //scim diff --git a/acceptance/selftest/record_proxy_retry/script b/acceptance/selftest/record_proxy_retry/script new file mode 100644 index 00000000000..10206fc75a0 --- /dev/null +++ b/acceptance/selftest/record_proxy_retry/script @@ -0,0 +1,11 @@ +title "A request answered with a transient error is left out of the recorded requests; the CLI's repeat is recorded" +fault.py "GET /api/2.0/postgres/projects/*/operations/*" 500 0 1 INTERNAL_ERROR +trace $CLI bundle deploy +trace print_requests.py --get //postgres +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "A request still failing when the CLI gives up is left out as well; the output shows the error" +fault.py "GET /api/2.0/preview/scim/v2/Me" 500 0 1 INTERNAL_ERROR +errcode trace $CLI current-user me +trace print_requests.py --get //scim diff --git a/acceptance/selftest/record_proxy_retry/test.toml b/acceptance/selftest/record_proxy_retry/test.toml new file mode 100644 index 00000000000..6af55345da1 --- /dev/null +++ b/acceptance/selftest/record_proxy_retry/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +Proxy = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" From 8598757c633f3d1689d85de82e55d611e9729540 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 13:20:02 +0000 Subject: [PATCH 2/3] acc: drop a failed attempt only when the CLI repeats it A request answered with a transient error is now removed from out.requests.txt only once the same request is recorded again, so a request the CLI does not repeat (e.g. a create that succeeded despite a 500 and was found by the recovery lookup) stays in the recording. cloud: TestAccept/selftest/record_cloud TestAccept/bundle/resources/postgres_roles/update TestAccept/bundle/resources/postgres_databases/basic Co-authored-by: Isaac --- acceptance/internal/prepare_server.go | 61 +++++++++++-------- .../selftest/record_proxy_retry/output.txt | 8 ++- acceptance/selftest/record_proxy_retry/script | 4 +- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index d6ace0c3a46..783f57b1469 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -228,8 +228,10 @@ type requestRecorder struct { mu sync.Mutex // entries holds what was written for requests still waiting for a response; nil unless - // dropRetried is used. + // failed attempts are dropped. entries map[*testserver.Request]string + // failed counts the entries answered with a transient error and not repeated yet. + failed map[string]int } func newRequestRecorder(t *testing.T, includeHeaders []string, outputDir string, dropRetried bool) *requestRecorder { @@ -240,6 +242,7 @@ func newRequestRecorder(t *testing.T, includeHeaders []string, outputDir string, } if dropRetried { r.entries = map[*testserver.Request]string{} + r.failed = map[string]int{} } return r } @@ -257,6 +260,11 @@ func (r *requestRecorder) record(request *testserver.Request) { assert.NoErrorf(r.t, err, "Failed to json-encode: %#v", req) entry := string(reqJson) + "\n" + if r.failed[entry] > 0 { + r.failed[entry]-- + r.dropFailedAttempt(request, entry) + } + f, err := os.OpenFile(r.path(), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) assert.NoError(r.t, err) defer f.Close() @@ -268,29 +276,12 @@ func (r *requestRecorder) record(request *testserver.Request) { } } -// isRetriedStatus reports whether the CLI retries a request answered with status (see -// libs/retry and the SDK's own retries). -func isRetriedStatus(status int) bool { - return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500 -} - -// dropRetried removes a request answered with a transient error from out.requests.txt. A real -// workspace answers with such errors now and then; the CLI repeats the request and the repeat is -// recorded, so dropping the failed attempt keeps the goldens stable. Only the proxy uses it: the -// testserver answers with an error only when a test injects one, and its goldens show that. -func (r *requestRecorder) dropRetried(request *testserver.Request, response *testserver.EncodedResponse) { - r.mu.Lock() - defer r.mu.Unlock() - - entry, ok := r.entries[request] - delete(r.entries, request) - if !ok || !isRetriedStatus(response.StatusCode) { - return - } - +// dropFailedAttempt removes the last copy of entry from out.requests.txt: the attempt that +// request repeats. +func (r *requestRecorder) dropFailedAttempt(request *testserver.Request, entry string) { data, err := os.ReadFile(r.path()) if errors.Is(err, os.ErrNotExist) { - // The script removed the file after the request was sent. + // The script consumed the file since the failed attempt. return } require.NoError(r.t, err) @@ -301,7 +292,29 @@ func (r *requestRecorder) dropRetried(request *testserver.Request, response *tes } data = append(data[:i:i], data[i+len(entry):]...) require.NoError(r.t, os.WriteFile(r.path(), data, 0o644)) - r.t.Logf("Dropped from out.requests.txt after %d response: %s %s", response.StatusCode, request.Method, request.URL.Path) + r.t.Logf("Dropped from out.requests.txt, repeated after a transient error: %s %s", request.Method, request.URL.Path) +} + +// isTransientStatus reports whether a response status is one the CLI may repeat a request after +// (see libs/retry and the SDK's own retries). +func isTransientStatus(status int) bool { + return status == http.StatusRequestTimeout || status == http.StatusTooManyRequests || status >= 500 +} + +// noteResponse remembers a request answered with a transient error, so that if the CLI repeats +// it, the failed attempt is dropped from out.requests.txt. A real workspace answers with such +// errors now and then, so recording both attempts makes request goldens flaky. A request that is +// not repeated stays recorded. Only the proxy uses this: the testserver answers with an error only +// when a test injects one, and its goldens show that. +func (r *requestRecorder) noteResponse(request *testserver.Request, response *testserver.EncodedResponse) { + r.mu.Lock() + defer r.mu.Unlock() + + entry, ok := r.entries[request] + delete(r.entries, request) + if ok && isTransientStatus(response.StatusCode) { + r.failed[entry]++ + } } func logResponseCallback(t *testing.T) func(request *testserver.Request, response *testserver.EncodedResponse) { @@ -391,7 +404,7 @@ func startProxyServer(t *testing.T, if recordRequests { recorder := newRequestRecorder(t, includeHeaders, outputDir, true) s.RequestCallback = recorder.record - responseCallbacks = append(responseCallbacks, recorder.dropRetried) + responseCallbacks = append(responseCallbacks, recorder.noteResponse) } // Log API responses if the -logrequests flag is set. diff --git a/acceptance/selftest/record_proxy_retry/output.txt b/acceptance/selftest/record_proxy_retry/output.txt index 74b36996c68..cebe35dff9f 100644 --- a/acceptance/selftest/record_proxy_retry/output.txt +++ b/acceptance/selftest/record_proxy_retry/output.txt @@ -1,5 +1,5 @@ -=== A request answered with a transient error is left out of the recorded requests; the CLI's repeat is recorded +=== A request answered with a transient error and repeated by the CLI is recorded once >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -26,7 +26,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "path": "/api/2.0/postgres/projects/test-pg-proj/operations/[UUID]" } -=== A request still failing when the CLI gives up is left out as well; the output shows the error +=== A request answered with a transient error and not repeated stays recorded >>> [CLI] current-user me Error: Fault injected by test. @@ -34,3 +34,7 @@ Error: Fault injected by test. Exit code: 1 >>> print_requests.py --get //scim +{ + "method": "GET", + "path": "/api/2.0/preview/scim/v2/Me" +} diff --git a/acceptance/selftest/record_proxy_retry/script b/acceptance/selftest/record_proxy_retry/script index 10206fc75a0..06e2b299221 100644 --- a/acceptance/selftest/record_proxy_retry/script +++ b/acceptance/selftest/record_proxy_retry/script @@ -1,11 +1,11 @@ -title "A request answered with a transient error is left out of the recorded requests; the CLI's repeat is recorded" +title "A request answered with a transient error and repeated by the CLI is recorded once" fault.py "GET /api/2.0/postgres/projects/*/operations/*" 500 0 1 INTERNAL_ERROR trace $CLI bundle deploy trace print_requests.py --get //postgres $CLI bundle destroy --auto-approve > /dev/null 2>&1 rm -f "$OUT_REQUESTS" -title "A request still failing when the CLI gives up is left out as well; the output shows the error" +title "A request answered with a transient error and not repeated stays recorded" fault.py "GET /api/2.0/preview/scim/v2/Me" 500 0 1 INTERNAL_ERROR errcode trace $CLI current-user me trace print_requests.py --get //scim From 04a240b67bfdb3fd797753b4b000889ec6dc2d10 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 16:06:41 +0000 Subject: [PATCH 3/3] Address review: write destroy output to LOG.destroy, assert with musterr cloud: TestAccept/selftest/record_cloud TestAccept/bundle/resources/postgres_roles/update TestAccept/bundle/resources/postgres_databases/basic Co-authored-by: Isaac --- acceptance/selftest/record_proxy_retry/output.txt | 4 +--- acceptance/selftest/record_proxy_retry/script | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/acceptance/selftest/record_proxy_retry/output.txt b/acceptance/selftest/record_proxy_retry/output.txt index cebe35dff9f..4980ba49a08 100644 --- a/acceptance/selftest/record_proxy_retry/output.txt +++ b/acceptance/selftest/record_proxy_retry/output.txt @@ -28,11 +28,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === A request answered with a transient error and not repeated stays recorded ->>> [CLI] current-user me +>>> musterr [CLI] current-user me Error: Fault injected by test. -Exit code: 1 - >>> print_requests.py --get //scim { "method": "GET", diff --git a/acceptance/selftest/record_proxy_retry/script b/acceptance/selftest/record_proxy_retry/script index 06e2b299221..6723d3a009c 100644 --- a/acceptance/selftest/record_proxy_retry/script +++ b/acceptance/selftest/record_proxy_retry/script @@ -2,10 +2,10 @@ title "A request answered with a transient error and repeated by the CLI is reco fault.py "GET /api/2.0/postgres/projects/*/operations/*" 500 0 1 INTERNAL_ERROR trace $CLI bundle deploy trace print_requests.py --get //postgres -$CLI bundle destroy --auto-approve > /dev/null 2>&1 +$CLI bundle destroy --auto-approve &> LOG.destroy rm -f "$OUT_REQUESTS" title "A request answered with a transient error and not repeated stays recorded" fault.py "GET /api/2.0/preview/scim/v2/Me" 500 0 1 INTERNAL_ERROR -errcode trace $CLI current-user me +trace musterr $CLI current-user me trace print_requests.py --get //scim