From 7b80619c828e3b314e42a74b0317804781b9b1d9 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 20 Nov 2026 08:12:40 +0000 Subject: [PATCH 1/2] acc: drop create-recovery lookups from cloud request recordings When a Lakebase create is answered with a transient error, the CLI reads the resource (404) before it repeats the create. The proxy dropped the failed POST but kept the lookup GET, so out.requests goldens still flaked on cloud. gcp: TestAccept/bundle/resources/postgres_roles TestAccept/bundle/resources/postgres_catalogs/basic aws: TestAccept/bundle/resources/postgres_roles/basic Co-authored-by: Isaac --- acceptance/internal/prepare_server.go | 64 ++++++++++++++++--- .../selftest/record_proxy_retry/output.txt | 29 ++++++++- acceptance/selftest/record_proxy_retry/script | 7 ++ 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 783f57b1469..6625a74687d 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -232,6 +232,20 @@ type requestRecorder struct { entries map[*testserver.Request]string // failed counts the entries answered with a transient error and not repeated yet. failed map[string]int + // createLookups maps the path of a resource that a create request answered with a transient + // error names (the request path plus one of its query values, e.g. /api/2.0/postgres/projects + // with project_id=p names /api/2.0/postgres/projects/p) to that request's entry. + createLookups map[string]string + // lookups holds, per failed create entry, the GETs of its resource answered with 404 since. The + // CLI makes them to check whether the failed request created the resource before it repeats + // the request, so they go together with the failed attempt. + lookups map[string][]recordedLookup +} + +// recordedLookup is a GET answered with 404 that a repeated create makes redundant. +type recordedLookup struct { + path string + entry string } func newRequestRecorder(t *testing.T, includeHeaders []string, outputDir string, dropRetried bool) *requestRecorder { @@ -243,6 +257,8 @@ func newRequestRecorder(t *testing.T, includeHeaders []string, outputDir string, if dropRetried { r.entries = map[*testserver.Request]string{} r.failed = map[string]int{} + r.createLookups = map[string]string{} + r.lookups = map[string][]recordedLookup{} } return r } @@ -262,7 +278,15 @@ func (r *requestRecorder) record(request *testserver.Request) { if r.failed[entry] > 0 { r.failed[entry]-- - r.dropFailedAttempt(request, entry) + if r.dropEntry(entry) { + r.t.Logf("Dropped from out.requests.txt, repeated after a transient error: %s %s", request.Method, request.URL.Path) + } + for _, lookup := range r.lookups[entry] { + if r.dropEntry(lookup.entry) { + r.t.Logf("Dropped from out.requests.txt, a lookup before a repeated create: GET %s", lookup.path) + } + } + delete(r.lookups, entry) } f, err := os.OpenFile(r.path(), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) @@ -276,23 +300,22 @@ func (r *requestRecorder) record(request *testserver.Request) { } } -// dropFailedAttempt removes the last copy of entry from out.requests.txt: the attempt that -// request repeats. -func (r *requestRecorder) dropFailedAttempt(request *testserver.Request, entry string) { +// dropEntry removes the last copy of entry from out.requests.txt and reports whether it did. +func (r *requestRecorder) dropEntry(entry string) bool { data, err := os.ReadFile(r.path()) if errors.Is(err, os.ErrNotExist) { - // The script consumed the file since the failed attempt. - return + // The script consumed the file since the entry was written. + return false } require.NoError(r.t, err) i := strings.LastIndex(string(data), entry) if i < 0 { - return + return false } data = append(data[:i:i], data[i+len(entry):]...) require.NoError(r.t, os.WriteFile(r.path(), data, 0o644)) - r.t.Logf("Dropped from out.requests.txt, repeated after a transient error: %s %s", request.Method, request.URL.Path) + return true } // isTransientStatus reports whether a response status is one the CLI may repeat a request after @@ -304,7 +327,9 @@ func isTransientStatus(status int) bool { // noteResponse remembers a request answered with a transient error, so that if the CLI repeats // it, the failed attempt is dropped from out.requests.txt. A real workspace answers with such // errors now and then, so recording both attempts makes request goldens flaky. A request that is -// not repeated stays recorded. Only the proxy uses this: the testserver answers with an error only +// not repeated stays recorded. Before it repeats a failed create, the CLI reads the resource to +// see whether the request created it anyway; a read answered with 404 is dropped with the failed +// attempt. A read that finds the resource stays recorded, since no repeat follows. Only the proxy uses this: the testserver answers with an error only // when a test injects one, and its goldens show that. func (r *requestRecorder) noteResponse(request *testserver.Request, response *testserver.EncodedResponse) { r.mu.Lock() @@ -312,8 +337,27 @@ func (r *requestRecorder) noteResponse(request *testserver.Request, response *te entry, ok := r.entries[request] delete(r.entries, request) - if ok && isTransientStatus(response.StatusCode) { + if !ok { + return + } + if isTransientStatus(response.StatusCode) { r.failed[entry]++ + if request.Method == http.MethodPost { + for _, values := range request.URL.Query() { + for _, value := range values { + if value != "" && !strings.Contains(value, "/") { + r.createLookups[request.URL.Path+"/"+value] = entry + } + } + } + } + return + } + if request.Method == http.MethodGet && response.StatusCode == http.StatusNotFound { + create, found := r.createLookups[request.URL.Path] + if found && r.failed[create] > 0 { + r.lookups[create] = append(r.lookups[create], recordedLookup{path: request.URL.Path, entry: entry}) + } } } diff --git a/acceptance/selftest/record_proxy_retry/output.txt b/acceptance/selftest/record_proxy_retry/output.txt index 4980ba49a08..3840f138e14 100644 --- a/acceptance/selftest/record_proxy_retry/output.txt +++ b/acceptance/selftest/record_proxy_retry/output.txt @@ -23,7 +23,34 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } { "method": "GET", - "path": "/api/2.0/postgres/projects/test-pg-proj/operations/[UUID]" + "path": "/api/2.0/postgres/projects/test-pg-proj/operations/[UUID][0]" +} + +=== A create answered with a transient error, looked up (404) and repeated by the CLI is recorded once + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj" + }, + "body": { + "spec": { + "display_name": "Test Postgres Project", + "pg_version": 16 + } + } +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj/operations/[UUID][1]" } === A request answered with a transient error and not repeated stays recorded diff --git a/acceptance/selftest/record_proxy_retry/script b/acceptance/selftest/record_proxy_retry/script index 6723d3a009c..b12474ddd14 100644 --- a/acceptance/selftest/record_proxy_retry/script +++ b/acceptance/selftest/record_proxy_retry/script @@ -5,6 +5,13 @@ trace print_requests.py --get //postgres $CLI bundle destroy --auto-approve &> LOG.destroy rm -f "$OUT_REQUESTS" +title "A create answered with a transient error, looked up (404) and repeated by the CLI is recorded once" +fault.py "POST /api/2.0/postgres/projects" 500 0 1 INTERNAL_ERROR +trace $CLI bundle deploy +trace print_requests.py --get //postgres +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + title "A request answered with a transient error and not repeated stays recorded" fault.py "GET /api/2.0/preview/scim/v2/Me" 500 0 1 INTERNAL_ERROR trace musterr $CLI current-user me From df963fe7e2eae344284fe30f9173aa46dcf8ee44 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 20 Nov 2026 08:40:50 +0000 Subject: [PATCH 2/2] acc: end create recovery when the lookup finds the resource Match lookups only on *_id query values, rewrite out.requests.txt once per repeat, and stop treating reads as lookups once the create is repeated or the resource is found, so a later identical create keeps its POST. gcp: TestAccept/bundle/resources/postgres_roles TestAccept/bundle/resources/postgres_catalogs/basic aws: TestAccept/bundle/resources/postgres_roles/basic Co-authored-by: Isaac --- acceptance/internal/prepare_server.go | 83 ++++++++++++++----- .../selftest/record_proxy_retry/output.txt | 21 +++++ acceptance/selftest/record_proxy_retry/script | 9 ++ 3 files changed, 91 insertions(+), 22 deletions(-) diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 6625a74687d..57904951238 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -278,15 +278,22 @@ func (r *requestRecorder) record(request *testserver.Request) { if r.failed[entry] > 0 { r.failed[entry]-- - if r.dropEntry(entry) { + lookups := r.lookups[entry] + delete(r.lookups, entry) + r.forgetCreateLookups(entry) + entries := []string{entry} + for _, lookup := range lookups { + entries = append(entries, lookup.entry) + } + dropped := r.dropEntries(entries) + if dropped[0] { r.t.Logf("Dropped from out.requests.txt, repeated after a transient error: %s %s", request.Method, request.URL.Path) } - for _, lookup := range r.lookups[entry] { - if r.dropEntry(lookup.entry) { + for i, lookup := range lookups { + if dropped[i+1] { r.t.Logf("Dropped from out.requests.txt, a lookup before a repeated create: GET %s", lookup.path) } } - delete(r.lookups, entry) } f, err := os.OpenFile(r.path(), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) @@ -300,22 +307,39 @@ func (r *requestRecorder) record(request *testserver.Request) { } } -// dropEntry removes the last copy of entry from out.requests.txt and reports whether it did. -func (r *requestRecorder) dropEntry(entry string) bool { +// dropEntries removes the last copy of each entry from out.requests.txt and reports which ones it +// found. +func (r *requestRecorder) dropEntries(entries []string) []bool { + dropped := make([]bool, len(entries)) data, err := os.ReadFile(r.path()) if errors.Is(err, os.ErrNotExist) { - // The script consumed the file since the entry was written. - return false + // The script consumed the file since the entries were written. + return dropped } require.NoError(r.t, err) - i := strings.LastIndex(string(data), entry) - if i < 0 { - return false + for n, entry := range entries { + i := strings.LastIndex(string(data), entry) + if i < 0 { + continue + } + data = append(data[:i:i], data[i+len(entry):]...) + dropped[n] = true + } + if slices.Contains(dropped, true) { + require.NoError(r.t, os.WriteFile(r.path(), data, 0o644)) + } + return dropped +} + +// forgetCreateLookups stops treating reads as lookups for the failed create entry: the CLI +// repeated the create or found the resource, so its recovery is over. +func (r *requestRecorder) forgetCreateLookups(create string) { + for path, entry := range r.createLookups { + if entry == create { + delete(r.createLookups, path) + } } - data = append(data[:i:i], data[i+len(entry):]...) - require.NoError(r.t, os.WriteFile(r.path(), data, 0o644)) - return true } // isTransientStatus reports whether a response status is one the CLI may repeat a request after @@ -329,8 +353,9 @@ func isTransientStatus(status int) bool { // errors now and then, so recording both attempts makes request goldens flaky. A request that is // not repeated stays recorded. Before it repeats a failed create, the CLI reads the resource to // see whether the request created it anyway; a read answered with 404 is dropped with the failed -// attempt. A read that finds the resource stays recorded, since no repeat follows. Only the proxy uses this: the testserver answers with an error only -// when a test injects one, and its goldens show that. +// attempt. A read that finds the resource stays recorded, since no repeat follows, and ends the +// recovery. Only the proxy uses this: the testserver answers with an error only when a test +// injects one, and its goldens show that. func (r *requestRecorder) noteResponse(request *testserver.Request, response *testserver.EncodedResponse) { r.mu.Lock() defer r.mu.Unlock() @@ -343,7 +368,11 @@ func (r *requestRecorder) noteResponse(request *testserver.Request, response *te if isTransientStatus(response.StatusCode) { r.failed[entry]++ if request.Method == http.MethodPost { - for _, values := range request.URL.Query() { + // The id of the resource a create names travels in a query parameter such as project_id. + for key, values := range request.URL.Query() { + if !strings.HasSuffix(key, "_id") { + continue + } for _, value := range values { if value != "" && !strings.Contains(value, "/") { r.createLookups[request.URL.Path+"/"+value] = entry @@ -353,12 +382,22 @@ func (r *requestRecorder) noteResponse(request *testserver.Request, response *te } return } - if request.Method == http.MethodGet && response.StatusCode == http.StatusNotFound { - create, found := r.createLookups[request.URL.Path] - if found && r.failed[create] > 0 { - r.lookups[create] = append(r.lookups[create], recordedLookup{path: request.URL.Path, entry: entry}) - } + if request.Method != http.MethodGet { + return + } + create, found := r.createLookups[request.URL.Path] + if !found || r.failed[create] == 0 { + return + } + if response.StatusCode == http.StatusNotFound { + r.lookups[create] = append(r.lookups[create], recordedLookup{path: request.URL.Path, entry: entry}) + return } + // The failed create did create the resource, so the CLI uses it and does not repeat the create: + // the failed attempt stands for the create, and later reads of the resource are not lookups. + r.failed[create]-- + delete(r.lookups, create) + r.forgetCreateLookups(create) } func logResponseCallback(t *testing.T) func(request *testserver.Request, response *testserver.EncodedResponse) { diff --git a/acceptance/selftest/record_proxy_retry/output.txt b/acceptance/selftest/record_proxy_retry/output.txt index 3840f138e14..75ed9a720b1 100644 --- a/acceptance/selftest/record_proxy_retry/output.txt +++ b/acceptance/selftest/record_proxy_retry/output.txt @@ -53,6 +53,27 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "path": "/api/2.0/postgres/projects/test-pg-proj/operations/[UUID][1]" } +=== A create that failed but created the resource stays recorded, and a later identical create is kept too + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get --oneline //postgres/projects ^//operations/ ^//branches +{"method": "POST", "path": "/api/2.0/postgres/projects", "q": {"project_id": "test-pg-proj"}, "body": {"spec": {"display_name": "Test Postgres Project", "pg_version": 16}}} +{"method": "GET", "path": "/api/2.0/postgres/projects/test-pg-proj"} +{"method": "GET", "path": "/api/2.0/postgres/projects/test-pg-proj"} +{"method": "DELETE", "path": "/api/2.0/postgres/projects/test-pg-proj"} +{"method": "POST", "path": "/api/2.0/postgres/projects", "q": {"project_id": "test-pg-proj"}, "body": {"spec": {"display_name": "Test Postgres Project", "pg_version": 16}}} + === A request answered with a transient error and not repeated stays recorded >>> musterr [CLI] current-user me diff --git a/acceptance/selftest/record_proxy_retry/script b/acceptance/selftest/record_proxy_retry/script index b12474ddd14..934adbc8fa7 100644 --- a/acceptance/selftest/record_proxy_retry/script +++ b/acceptance/selftest/record_proxy_retry/script @@ -12,6 +12,15 @@ trace print_requests.py --get //postgres $CLI bundle destroy --auto-approve &> LOG.destroy rm -f "$OUT_REQUESTS" +title "A create that failed but created the resource stays recorded, and a later identical create is kept too" +fault.py --after-handler "POST /api/2.0/postgres/projects" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +$CLI bundle destroy --auto-approve &> LOG.destroy +trace $CLI bundle deploy +trace print_requests.py --get --oneline //postgres/projects ^//operations/ ^//branches +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + title "A request answered with a transient error and not repeated stays recorded" fault.py "GET /api/2.0/preview/scim/v2/Me" 500 0 1 INTERNAL_ERROR trace musterr $CLI current-user me