feat(api): base64 request forms for non-UTF-8 paths in grep, archive and commit-pack

.github/workflows/fuzz-nightly.yml

@@ -52,6 +52,9 @@ jobs:
             ./internal/jobs:FuzzPackObjectCount
             ./internal/api:FuzzClientKey
             ./internal/api:FuzzCoreCursor
+            ./internal/api:FuzzDecodeRawPath
+            ./internal/api:FuzzDecodeRawPatterns
+            ./internal/api:FuzzCommitPackFilePath
             ./internal/api:FuzzApplyRejection
             ./internal/api:FuzzPatchModeError
             ./internal/sync:FuzzParsePushRejections

docs/explanation/streaming-commit-protocols.md

@@ -68,6 +68,11 @@ Lines 2..N — one or more blob chunk frames per `upsert` file, in order:
   get 400 `invalid`.
 - Clients normalize paths (leading slashes stripped); the backend should still reject
   `..` traversal and absolute paths defensively.
+- A file whose name is not valid UTF-8 sends `path_base64`, the base64 of the raw
+  bytes, in place of `path`: exactly one of the two, and an empty one counts as absent.
+  A bad value gets 400 `invalid`
+  ([path encoding](../reference/http-api.md#conventions)). A replica older than
+  v0.20.0 does not read `path_base64` and answers 400 `invalid`.
 - Modes: `100644`, `100755`, `120000` (symlink), `160000` (submodule).
 
 ### Server semantics

docs/explanation/wire-compatibility.md

@@ -210,7 +210,7 @@ the [archived phase notes](../archive/phases/)):
 | `GET .../file` and `POST .../archive`: no documented file-size cap and no `422`; `archive` has only the caller-chosen `max_blob_size` | `file` and `archive` answer 422 for a file over 100 MiB (`archive` can skip such files with `max_blob_size`), before any response byte. The other read-side git [resource limits](../reference/http-api.md#resource-limits-on-git-reads) apply too. After an `archive` response has started, a limit closes the connection instead of returning a status | Hardening ([ADR-49](../adr/049-read-side-git-children-run-under-one-per-request-budget.md)): the limits bound server memory, temporary disk and delivery time for data that anyone who can push controls. Fetch larger files with git |
 | Every `503` response can carry an optional `Retry-After` header, the minimum delay in seconds before a retry (spec 2026-10-21) | Two 503s leave `Retry-After` out: a git read that exceeds its [time budget](../reference/http-api.md#resource-limits-on-git-reads), and a sync create whose upstream link was removed before its first sync was scheduled ([pending creates](../reference/http-api.md#repository-lifecycle-repowrite--orgread)). The 503 for exhausted git concurrency and the 503s for a create that is not finished carry `Retry-After` | A retry of either request fails the same way, so a `Retry-After` would invite a useless retry. The same page needs the same git time ([ADR-49](../adr/049-read-side-git-children-run-under-one-per-request-budget.md)). The pending create stays pending until `DELETE /repos/delete` clears the name ([#388](https://github.com/coder/openstorage/issues/388)) |
 | A token that fails verification (malformed, bad signature, expired, unknown issuer): upstream's live docs and servers answer `403` `forbidden` (`Invalid or expired token`) on the REST API, and `403` on the Git and LFS transports too (checked 2026-11-04). The vendored spec documents no status for it | The REST API answers `403` `forbidden` (`Invalid or expired token`), as upstream does ([#664](https://github.com/coder/openstorage/issues/664)). The Git and LFS transports answer `401` with `WWW-Authenticate: Basic`, as for a missing token ([authentication](../reference/authentication.md#failure-mapping)) | Git asks for credentials only after a `401`, and only a `401` makes it erase a stored credential. After a `403`, git fails and keeps the expired token in the credential helper, so every later command fails the same way until the user removes the token |
-| Every path field and `path` parameter is a plain `type: string`. The spec says nothing about tree entry names whose bytes are not valid UTF-8 | A response path that is not valid UTF-8 still lists with one U+FFFD per invalid byte, and gets an extra sibling `<field>_base64` with the raw bytes in standard base64 (for example `path_base64`, `paths_base64`, `old_path_base64`, `original_path_base64`). The sibling is absent for valid UTF-8 paths, so those responses are unchanged. This covers `files`, `files/metadata`, the diff routes, `blame`, `grep` and the merge conflict paths (`conflict_paths_base64` and `path_base64`). JSON request bodies cannot name such a file yet. The `path` query parameter takes the raw bytes percent-encoded ([path encoding](../reference/http-api.md#conventions)) | JSON strings hold Unicode text, so such a byte has no exact JSON form. A sibling field (like GitHub's `pathRaw`) is additive: no SDK reads it, and the SDKs ignore response keys they do not know. Upstream's schemas for these responses are closed (`additionalProperties: false`), so a client that validates a response against the vendored spec as written rejects one that has a sibling. No SDK validates that way. If upstream documents another form, we carry both ([#587](https://github.com/coder/openstorage/issues/587)) |
+| Every path field and `path` parameter is a plain `type: string`. The spec says nothing about tree entry names whose bytes are not valid UTF-8 | A response path that is not valid UTF-8 still lists with one U+FFFD per invalid byte, and gets an extra sibling `<field>_base64` with the raw bytes in standard base64 (for example `path_base64`, `paths_base64`, `old_path_base64`, `original_path_base64`). The sibling is absent for valid UTF-8 paths, so those responses are unchanged. This covers `files`, `files/metadata`, the diff routes, `blame`, `grep` and the merge conflict paths (`conflict_paths_base64` and `path_base64`). A JSON request body names such a path with `paths_base64` and `include_globs_base64`/`exclude_globs_base64` (grep and archive) or `path_base64` (a commit-pack file operation), from v0.20.0 on. An older replica ignores the request fields: grep and archive apply only the plain filters, so the result can hold more or fewer files, and commit-pack answers 400. The `path` query parameter takes the raw bytes percent-encoded ([path encoding](../reference/http-api.md#conventions)) | JSON strings hold Unicode text, so such a byte has no exact JSON form. A sibling field (like GitHub's `pathRaw`) is additive: no SDK reads it, and the SDKs ignore response keys they do not know. Upstream's schemas for these responses are closed (`additionalProperties: false`), so a client that validates a response against the vendored spec as written rejects one that has a sibling. No SDK validates that way. If upstream documents another form, we carry both ([#587](https://github.com/coder/openstorage/issues/587)) |
 
 Extensions that upstream's spec does not list:
 
@@ -231,6 +231,11 @@ Extensions that upstream's spec does not list:
   `paths_base64`, `old_path_base64`, `original_path_base64`, `conflict_paths_base64`),
   present only for a path that is not valid UTF-8
   ([path encoding](../reference/http-api.md#conventions));
+- the matching request fields: `paths_base64` and `file_filters.include_globs_base64` and
+  `exclude_globs_base64` on `POST .../grep`, `include_globs_base64` and
+  `exclude_globs_base64` on `POST .../archive`, and `files[].path_base64` on
+  `POST .../commit-pack`. Upstream's request schemas for these are closed too, so
+  upstream's own servers reject them;
 - the unauthenticated `/healthz` liveness probe.
 
 ## How drift is tracked

docs/reference/http-api.md

@@ -133,11 +133,29 @@ are in [`../upstream/openapi.yaml`](../upstream/openapi.yaml).
   raw bytes percent-encoded in the `path` query parameter, for example
   `file?path=a%FFb`. `file`, `files`, `files/metadata`, `commits`, `diff`,
   `branches/diff` and `blame` read `path` byte for byte. An `archive` (tar member
-  names) or a git clone gives the raw names. A JSON request body cannot hold such a
-  name yet: grep `paths` and commit-pack file paths cannot name the file, and the
-  archive and grep globs match it only through a wildcard (`a?b`, `a*`), which can
-  match other names too. Upstream documents no path encoding, so the siblings are an
-  extension ([#587](https://github.com/coder/openstorage/issues/587)).
+  names) or a git clone gives the raw names.
+
+  A JSON request body names such a path with a base64 form of the field:
+  - `grep`: `paths_base64` adds prefixes to `paths`, and `file_filters` takes
+    `include_globs_base64` and `exclude_globs_base64` next to the plain globs.
+  - `archive`: `include_globs_base64` and `exclude_globs_base64`.
+  - `commit-pack`: a file operation sends `path_base64` instead of `path`. It has exactly
+    one of the two, and an empty one counts as absent.
+
+  A value must be canonical base64 as in the siblings: standard alphabet, padding, no
+  line breaks. In grep and archive, a value that is not, or that decodes to a blank name
+  or one with a NUL byte, is a 400 with an `invalid_params` entry that names it
+  (`paths_base64.1`). In commit-pack, a value that is not canonical base64 is a 400
+  `invalid` with a message, and a decoded path goes through the checks of `path`. A
+  decoded glob matches the
+  raw bytes: an invalid byte matches only itself, `?` matches one byte of such a name,
+  and `*` matches any run. A `[...]` class that holds an invalid byte is malformed and
+  matches nothing. A replica older than v0.20.0 ignores the grep and archive fields, so
+  it applies only the plain `paths` and globs of the request: the result can hold more
+  files (only a base64 field was sent) or fewer (a plain filter was sent next to it). It
+  answers a commit-pack `path_base64` with 400 `invalid`. Upstream documents no path
+  encoding, so the siblings and these fields are an extension
+  ([#587](https://github.com/coder/openstorage/issues/587)).
 - **No client retries**: no SDK retries; every error surfaces immediately. Python
   client-side timeouts: 30s default, 60s diff/grep, 180s
   branch-create/merge/merge-preview/commit/tag/restore-commit.
@@ -341,8 +359,8 @@ or NUL fails the sync as a configuration error.
 | `HEAD /repos/file` | same | headers only |
 | `GET /repos/files` | `ref, ephemeral, path, recursive, cursor, limit` | `{paths: [..], paths_base64?: [..], ref, entries: [{path, path_base64?, type: blob\|tree\|symlink\|submodule, mode}], next_cursor, has_more}` — trees have no trailing `/`; branch on `type`. Without `recursive`, with an empty `recursive=`, or with `recursive=true`, it lists the contents of every folder under `path` and no `tree` entries. `recursive=false` lists only the direct children of `path`, folders as `tree` entries. No `cursor` and no `limit` (or `limit=0`): every entry, or 422 when the listing passes the git [output limit](#resource-limits-on-git-reads). Big trees must page: a recursive call with `limit` or `cursor` reads the tree only up to the end of its page, so a deep cursor takes longer than the first page but needs no more memory. A tree that git stores out of order (one that `git fsck` rejects as `treeNotSorted`) can page in git's order and skip entries; git itself cannot look such entries up by path |
 | `GET /repos/files/metadata` | same | `{files: [{path, path_base64?, mode, size, last_commit_sha, type}], commits: {<sha>: {author, date, message, order}}, ref, next_cursor, has_more}`. `order` is described below the table. `message` is the subject line only, as in the commits list ([Commits](#commits)). No `cursor` and no `limit` (or `limit=0`): every file. **429** when the files under `path` have more than 50,000 distinct last commits, also with `limit` or `cursor`: a retry gets the same 429 (no `Retry-After`), so send a narrower `path`. **404 on unborn branch** |
-| `POST /repos/archive` | body `{ref?, include_globs?, exclude_globs?, max_blob_size?, archive?: {prefix}}` | tar.gz stream |
-| `POST /repos/grep` | body `{query: {pattern, case_sensitive?}, ref?, ephemeral?, paths?, file_filters?: {include_globs, exclude_globs, extension_filters}, context?: {before, after}, limits?: {max_lines, max_matches_per_file}, pagination?: {cursor, limit}}` | `{query, repo: {ref, commit}, matches: [{path, path_base64?, lines: [{line_number, text, type: "match"\|context}]}], next_cursor, has_more}` |
+| `POST /repos/archive` | body `{ref?, include_globs?, include_globs_base64?, exclude_globs?, exclude_globs_base64?, max_blob_size?, archive?: {prefix}}` | tar.gz stream |
+| `POST /repos/grep` | body `{query: {pattern, case_sensitive?}, ref?, ephemeral?, paths?, paths_base64?, file_filters?: {include_globs, include_globs_base64?, exclude_globs, exclude_globs_base64?, extension_filters}, context?: {before, after}, limits?: {max_lines, max_matches_per_file}, pagination?: {cursor, limit}}` | `{query, repo: {ref, commit}, matches: [{path, path_base64?, lines: [{line_number, text, type: "match"\|context}]}], next_cursor, has_more}` |
 | `GET /repos/blame` | `path` (req), `ref`, `ephemeral`, `range` (repeatable, `git blame -L` specs like `1,10`), `detect_moves` | `{ref, path, path_base64?, commit_sha, lines: [{line_number, commit_sha, original_line_number, original_path, original_path_base64?, previous_commit_sha?, author_name, author_email, author_time, committer_name, committer_email, committer_time, summary}]}` |
 
 `order` in `files/metadata` is the commit's history-order rank relative to the resolved ref: its 0-based position in `git rev-list --topo-order <resolved commit>`. The ref's own commit is `0`, and larger values are older. A commit always ranks higher than every one of its descendants, whatever its author or committer date. Among a folder's files, the one with the smallest `order` was changed most recently in the ref's history.

internal/api/fuzz_test.go

@@ -157,6 +157,88 @@ func FuzzRawPathBase64(f *testing.F) {
 	})
 }
 
+// FuzzDecodeRawPath: a request's path_base64 text decodes only when it is
+// the canonical padded standard base64 of the bytes it yields, so no two
+// texts name one path.
+func FuzzDecodeRawPath(f *testing.F) {
+	for _, seed := range []string{"", "YQ==", "YQ=", "YQ", "YR==", "Yf9i", "!!", "Yf9\ni", "Yf9i\r\n", "-_8=", "+/8=", "AA==", "////", "Y W I=", "YQ==YQ=="} {
+		f.Add(seed)
+	}
+	f.Fuzz(func(t *testing.T, s string) {
+		p, ok := decodeRawPath(s)
+		if ok != (encodeRawPath(p) == s) || (!ok && p != "") {
+			t.Fatalf("decodeRawPath(%q) = %q, %v", s, p, ok)
+		}
+		// Every byte string has one text, and it decodes back.
+		if q, ok := decodeRawPath(encodeRawPath(s)); !ok || q != s {
+			t.Fatalf("decodeRawPath(encodeRawPath(%q)) = %q, %v", s, q, ok)
+		}
+	})
+}
+
+// FuzzDecodeRawPatterns: a decoded list holds one value per element, each
+// not blank and with no NUL, and a failure names the first bad element.
+func FuzzDecodeRawPatterns(f *testing.F) {
+	for _, seed := range [][2]string{
+		{"YQ==", "Yf9i"}, {"", "YQ=="}, {"YQ==", ""}, {"AA==", "YQ=="}, {"IA==", "YQ=="},
+		{"CQ==", "Yf9i"}, {"!!", "!!"}, {"YR==", "YQ=="}, {"YQ==", "YQ=="},
+	} {
+		f.Add(seed[0], seed[1])
+	}
+	f.Fuzz(func(t *testing.T, a, b string) {
+		list := []string{a, b}
+		decoded, bad := decodeRawPatterns(list)
+		valid := func(enc string) bool {
+			p, ok := decodeRawPath(enc)
+			return ok && strings.TrimSpace(p) != "" && !strings.ContainsRune(p, 0)
+		}
+		if bad < 0 {
+			if len(decoded) != 2 || encodeRawPath(decoded[0]) != a || encodeRawPath(decoded[1]) != b || !valid(a) || !valid(b) {
+				t.Fatalf("decodeRawPatterns(%q) = %q, %d", list, decoded, bad)
+			}
+			return
+		}
+		if decoded != nil || bad > 1 || valid(list[bad]) || (bad == 1 && !valid(a)) {
+			t.Fatalf("decodeRawPatterns(%q) = %q, %d", list, decoded, bad)
+		}
+	})
+}
+
+// FuzzCommitPackFilePath: a file operation names its path with exactly one
+// of path and path_base64 (an empty one is absent), and the path it ends with
+// is the one it names.
+func FuzzCommitPackFilePath(f *testing.F) {
+	for _, seed := range [][2]string{
+		{"a.txt", ""}, {"", "Yf9i"}, {"a.txt", "Yf9i"}, {"", ""}, {"", "YR=="}, {"", "Li4v"},
+		{"", "LmdpdC9jb25maWc="}, {"", "YQBi"}, {"", "AA=="}, {"../a", ""}, {"a\xffb", ""},
+	} {
+		f.Add(seed[0], seed[1])
+	}
+	f.Fuzz(func(t *testing.T, path, enc string) {
+		files := []wcFileOp{{Path: path, PathBase64: enc, ContentID: "c", Operation: "upsert"}}
+		err := wcValidateFiles(files)
+		decoded, decodedOK := decodeRawPath(enc)
+		want := path
+		if enc != "" {
+			want = decoded
+		}
+		switch {
+		case enc != "" && (path != "" || !decodedOK):
+			if err == nil {
+				t.Fatalf("path %q, path_base64 %q: accepted", path, enc)
+			}
+		case err == nil:
+			if files[0].Path != want || wcValidPath(want) != nil {
+				t.Fatalf("path %q, path_base64 %q: resolved to %q, want the valid path %q", path, enc, files[0].Path, want)
+			}
+		default:
+			if wcValidPath(want) == nil {
+				t.Fatalf("path %q, path_base64 %q: refused with %v though %q is valid", path, enc, err, want)
+			}
+		}
+	})
+}
+
 func FuzzCoreCursor(f *testing.F) {
 	f.Add(coreCursorEncode(cursorKindCommits, "abc123", "2024-01-01T00:00:00Z"), "main", "src/a|b.txt")
 	f.Add("", "", "")