Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 98 additions & 15 deletions acceptance/internal/prepare_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,20 @@ type requestRecorder struct {
entries map[*testserver.Request]string
// failed counts the entries answered with a transient error and not repeated yet.
failed map[string]int
// createLookups maps the path of a resource that a create request answered with a transient
// error names (the request path plus one of its query values, e.g. /api/2.0/postgres/projects
// with project_id=p names /api/2.0/postgres/projects/p) to that request's entry.
createLookups map[string]string
// lookups holds, per failed create entry, the GETs of its resource answered with 404 since. The
// CLI makes them to check whether the failed request created the resource before it repeats
// the request, so they go together with the failed attempt.
lookups map[string][]recordedLookup
}

// recordedLookup is a GET answered with 404 that a repeated create makes redundant.
type recordedLookup struct {
path string
entry string
}

func newRequestRecorder(t *testing.T, includeHeaders []string, outputDir string, dropRetried bool) *requestRecorder {
Expand All @@ -243,6 +257,8 @@ func newRequestRecorder(t *testing.T, includeHeaders []string, outputDir string,
if dropRetried {
r.entries = map[*testserver.Request]string{}
r.failed = map[string]int{}
r.createLookups = map[string]string{}
r.lookups = map[string][]recordedLookup{}
}
return r
}
Expand All @@ -262,7 +278,22 @@ func (r *requestRecorder) record(request *testserver.Request) {

if r.failed[entry] > 0 {
r.failed[entry]--
r.dropFailedAttempt(request, entry)
lookups := r.lookups[entry]
delete(r.lookups, entry)
r.forgetCreateLookups(entry)
entries := []string{entry}
for _, lookup := range lookups {
entries = append(entries, lookup.entry)
}
dropped := r.dropEntries(entries)
if dropped[0] {
r.t.Logf("Dropped from out.requests.txt, repeated after a transient error: %s %s", request.Method, request.URL.Path)
}
for i, lookup := range lookups {
if dropped[i+1] {
r.t.Logf("Dropped from out.requests.txt, a lookup before a repeated create: GET %s", lookup.path)
}
}
}

f, err := os.OpenFile(r.path(), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644)
Expand All @@ -276,23 +307,39 @@ func (r *requestRecorder) record(request *testserver.Request) {
}
}

// dropFailedAttempt removes the last copy of entry from out.requests.txt: the attempt that
// request repeats.
func (r *requestRecorder) dropFailedAttempt(request *testserver.Request, entry string) {
// dropEntries removes the last copy of each entry from out.requests.txt and reports which ones it
// found.
func (r *requestRecorder) dropEntries(entries []string) []bool {
dropped := make([]bool, len(entries))
data, err := os.ReadFile(r.path())
if errors.Is(err, os.ErrNotExist) {
// The script consumed the file since the failed attempt.
return
// The script consumed the file since the entries were written.
return dropped
}
require.NoError(r.t, err)

i := strings.LastIndex(string(data), entry)
if i < 0 {
return
for n, entry := range entries {
i := strings.LastIndex(string(data), entry)
if i < 0 {
continue
}
data = append(data[:i:i], data[i+len(entry):]...)
dropped[n] = true
}
if slices.Contains(dropped, true) {
require.NoError(r.t, os.WriteFile(r.path(), data, 0o644))
}
return dropped
}

// forgetCreateLookups stops treating reads as lookups for the failed create entry: the CLI
// repeated the create or found the resource, so its recovery is over.
func (r *requestRecorder) forgetCreateLookups(create string) {
for path, entry := range r.createLookups {
if entry == create {
delete(r.createLookups, path)
}
}
data = append(data[:i:i], data[i+len(entry):]...)
require.NoError(r.t, os.WriteFile(r.path(), data, 0o644))
r.t.Logf("Dropped from out.requests.txt, repeated after a transient error: %s %s", request.Method, request.URL.Path)
}

// isTransientStatus reports whether a response status is one the CLI may repeat a request after
Expand All @@ -304,17 +351,53 @@ func isTransientStatus(status int) bool {
// noteResponse remembers a request answered with a transient error, so that if the CLI repeats
// it, the failed attempt is dropped from out.requests.txt. A real workspace answers with such
// errors now and then, so recording both attempts makes request goldens flaky. A request that is
// not repeated stays recorded. Only the proxy uses this: the testserver answers with an error only
// when a test injects one, and its goldens show that.
// not repeated stays recorded. Before it repeats a failed create, the CLI reads the resource to
// see whether the request created it anyway; a read answered with 404 is dropped with the failed
// attempt. A read that finds the resource stays recorded, since no repeat follows, and ends the
// recovery. Only the proxy uses this: the testserver answers with an error only when a test
// injects one, and its goldens show that.
func (r *requestRecorder) noteResponse(request *testserver.Request, response *testserver.EncodedResponse) {
r.mu.Lock()
defer r.mu.Unlock()

entry, ok := r.entries[request]
delete(r.entries, request)
if ok && isTransientStatus(response.StatusCode) {
if !ok {
return
}
if isTransientStatus(response.StatusCode) {
r.failed[entry]++
if request.Method == http.MethodPost {
// The id of the resource a create names travels in a query parameter such as project_id.
for key, values := range request.URL.Query() {
if !strings.HasSuffix(key, "_id") {
continue
}
for _, value := range values {
if value != "" && !strings.Contains(value, "/") {
r.createLookups[request.URL.Path+"/"+value] = entry
}
}
}
}
return
}
if request.Method != http.MethodGet {
return
}
create, found := r.createLookups[request.URL.Path]
if !found || r.failed[create] == 0 {
return
}
if response.StatusCode == http.StatusNotFound {
r.lookups[create] = append(r.lookups[create], recordedLookup{path: request.URL.Path, entry: entry})
return
}
// The failed create did create the resource, so the CLI uses it and does not repeat the create:
// the failed attempt stands for the create, and later reads of the resource are not lookups.
r.failed[create]--
delete(r.lookups, create)
r.forgetCreateLookups(create)
}

func logResponseCallback(t *testing.T) func(request *testserver.Request, response *testserver.EncodedResponse) {
Expand Down
50 changes: 49 additions & 1 deletion acceptance/selftest/record_proxy_retry/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,57 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
}
{
"method": "GET",
"path": "/api/2.0/postgres/projects/test-pg-proj/operations/[UUID]"
"path": "/api/2.0/postgres/projects/test-pg-proj/operations/[UUID][0]"
}

=== A create answered with a transient error, looked up (404) and repeated by the CLI is recorded once

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Files: 0 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres
{
"method": "POST",
"path": "/api/2.0/postgres/projects",
"q": {
"project_id": "test-pg-proj"
},
"body": {
"spec": {
"display_name": "Test Postgres Project",
"pg_version": 16
}
}
}
{
"method": "GET",
"path": "/api/2.0/postgres/projects/test-pg-proj/operations/[UUID][1]"
}

=== A create that failed but created the resource stays recorded, and a later identical create is kept too

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Files: 0 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Files: 0 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get --oneline //postgres/projects ^//operations/ ^//branches
{"method": "POST", "path": "/api/2.0/postgres/projects", "q": {"project_id": "test-pg-proj"}, "body": {"spec": {"display_name": "Test Postgres Project", "pg_version": 16}}}
{"method": "GET", "path": "/api/2.0/postgres/projects/test-pg-proj"}
{"method": "GET", "path": "/api/2.0/postgres/projects/test-pg-proj"}
{"method": "DELETE", "path": "/api/2.0/postgres/projects/test-pg-proj"}
{"method": "POST", "path": "/api/2.0/postgres/projects", "q": {"project_id": "test-pg-proj"}, "body": {"spec": {"display_name": "Test Postgres Project", "pg_version": 16}}}

=== A request answered with a transient error and not repeated stays recorded

>>> musterr [CLI] current-user me
Expand Down
16 changes: 16 additions & 0 deletions acceptance/selftest/record_proxy_retry/script
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,22 @@ trace print_requests.py --get //postgres
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "A create answered with a transient error, looked up (404) and repeated by the CLI is recorded once"
fault.py "POST /api/2.0/postgres/projects" 500 0 1 INTERNAL_ERROR
trace $CLI bundle deploy
trace print_requests.py --get //postgres
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "A create that failed but created the resource stays recorded, and a later identical create is kept too"
fault.py --after-handler "POST /api/2.0/postgres/projects" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
$CLI bundle destroy --auto-approve &> LOG.destroy
trace $CLI bundle deploy
trace print_requests.py --get --oneline //postgres/projects ^//operations/ ^//branches
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "A request answered with a transient error and not repeated stays recorded"
fault.py "GET /api/2.0/preview/scim/v2/Me" 500 0 1 INTERNAL_ERROR
trace musterr $CLI current-user me
Expand Down
Loading