diff --git a/.changeset/edit-diff-bounds.md b/.changeset/edit-diff-bounds.md new file mode 100644 index 00000000..46420e8d --- /dev/null +++ b/.changeset/edit-diff-bounds.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +The `edit` tool bounds its diff with `maxDiffLines` and `maxDiffBytes` and reports `diffTruncated`; see [`edit` tool documentation](https://github.com/cloudflare/computer/blob/main/docs/09_tool_interface.md#edit). diff --git a/.changeset/exec-input-object.md b/.changeset/exec-input-object.md new file mode 100644 index 00000000..a49b13e3 --- /dev/null +++ b/.changeset/exec-input-object.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": minor +--- + +The `exec` tool's `input` is an object rather than any JSON value; see [`exec` tool documentation](https://github.com/cloudflare/computer/blob/main/docs/09_tool_interface.md#exec). diff --git a/.changeset/exec-input-repair.md b/.changeset/exec-input-repair.md new file mode 100644 index 00000000..8cccb2b7 --- /dev/null +++ b/.changeset/exec-input-repair.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +The `exec` tool parses an `input` object sent as a string of JSON; see [`exec` tool documentation](https://github.com/cloudflare/computer/blob/main/docs/09_tool_interface.md#exec). diff --git a/.changeset/exec-output-option.md b/.changeset/exec-output-option.md new file mode 100644 index 00000000..56624c6a --- /dev/null +++ b/.changeset/exec-output-option.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Tool sets take `execOutput: { maxLines, maxBytes }` for the `exec` tool's output limits; see [tool option documentation](https://github.com/cloudflare/computer/blob/main/docs/09_tool_interface.md#options). diff --git a/.changeset/find-brace-globs.md b/.changeset/find-brace-globs.md new file mode 100644 index 00000000..54393cf5 --- /dev/null +++ b/.changeset/find-brace-globs.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/dofs": patch +--- + +`find` and `grep` globs match brace alternatives such as `**/*.{ts,tsx}`; see [`find` documentation](https://github.com/cloudflare/computer/blob/main/docs/04_filesystem_interface.md#find). diff --git a/.changeset/find-prune-trailing-globstar.md b/.changeset/find-prune-trailing-globstar.md new file mode 100644 index 00000000..11dbc4eb --- /dev/null +++ b/.changeset/find-prune-trailing-globstar.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/dofs": minor +--- + +An exclusion ending in `/**` also prunes the directory it names in `find` and `grep`; see [`find` documentation](https://github.com/cloudflare/computer/blob/main/docs/04_filesystem_interface.md#find). diff --git a/.changeset/module-host-assets.md b/.changeset/module-host-assets.md new file mode 100644 index 00000000..66fe0bbc --- /dev/null +++ b/.changeset/module-host-assets.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Host module factories receive the Workspace's assets client as `host.assets`; see [host module documentation](https://github.com/cloudflare/computer/blob/main/docs/17_isolate_javascript.md#host-modules). diff --git a/.changeset/module-lazy-descriptions.md b/.changeset/module-lazy-descriptions.md new file mode 100644 index 00000000..c22aacfe --- /dev/null +++ b/.changeset/module-lazy-descriptions.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +`WorkerJavaScriptBackend` reads each host module's description whenever it describes itself, so a factory can describe something built later; see [module documentation](https://github.com/cloudflare/computer/blob/main/docs/17_isolate_javascript.md#modules). diff --git a/.changeset/pi-tool-details.md b/.changeset/pi-tool-details.md new file mode 100644 index 00000000..061fb0e6 --- /dev/null +++ b/.changeset/pi-tool-details.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +`createPiTools` results carry each tool's own output as `details`; see [pi documentation](https://github.com/cloudflare/computer/blob/main/docs/09_tool_interface.md#pi). diff --git a/.changeset/tool-glob-lists.md b/.changeset/tool-glob-lists.md new file mode 100644 index 00000000..98863be0 --- /dev/null +++ b/.changeset/tool-glob-lists.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +The `grep` tool's `include` and `exclude`, and the `find` tool's `exclude`, take one glob or a list; see [`grep` tool documentation](https://github.com/cloudflare/computer/blob/main/docs/09_tool_interface.md#grep). diff --git a/.changeset/tools-core-entry-point.md b/.changeset/tools-core-entry-point.md new file mode 100644 index 00000000..bd0e9988 --- /dev/null +++ b/.changeset/tools-core-entry-point.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Add `@cloudflare/computer/tools/core`, which exports `defineExec` without an agent library; see [tool documentation](https://github.com/cloudflare/computer/blob/main/docs/09_tool_interface.md#what-ships). diff --git a/.changeset/tools-root-option.md b/.changeset/tools-root-option.md new file mode 100644 index 00000000..8ffda9aa --- /dev/null +++ b/.changeset/tools-root-option.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Tool sets take `root` to confine the file tools to one directory and refuse symbolic links; see [root documentation](https://github.com/cloudflare/computer/blob/main/docs/09_tool_interface.md#root). diff --git a/.changeset/ws-artifacts-tokens.md b/.changeset/ws-artifacts-tokens.md new file mode 100644 index 00000000..9d6173d3 --- /dev/null +++ b/.changeset/ws-artifacts-tokens.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +`ws:artifacts` adds `createToken` and `share` for minting git tokens and clone URLs; see [`ws:artifacts` documentation](https://github.com/cloudflare/computer/blob/main/docs/17_isolate_javascript.md#wsartifacts). diff --git a/.changeset/ws-assets-module.md b/.changeset/ws-assets-module.md new file mode 100644 index 00000000..aecd5a24 --- /dev/null +++ b/.changeset/ws-assets-module.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Add `ws:assets` so isolate JavaScript can publish a Workspace file to a time-limited URL; see [`ws:assets` documentation](https://github.com/cloudflare/computer/blob/main/docs/17_isolate_javascript.md#wsassets). diff --git a/.changeset/ws-container-max-lines.md b/.changeset/ws-container-max-lines.md new file mode 100644 index 00000000..eb81b956 --- /dev/null +++ b/.changeset/ws-container-max-lines.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +`createContainerModule({ maxOutputLines })` limits each `ws:container` stream by lines as well as bytes; see [`ws:container` documentation](https://github.com/cloudflare/computer/blob/main/docs/17_isolate_javascript.md#wscontainer). diff --git a/.changeset/ws-container-prelude.md b/.changeset/ws-container-prelude.md new file mode 100644 index 00000000..3ff9d671 --- /dev/null +++ b/.changeset/ws-container-prelude.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +`createContainerModule({ prelude })` runs shell text such as `set -o pipefail` before each `ws:container` command; see [`ws:container` documentation](https://github.com/cloudflare/computer/blob/main/docs/17_isolate_javascript.md#wscontainer). diff --git a/.changeset/ws-tools-module.md b/.changeset/ws-tools-module.md new file mode 100644 index 00000000..d3e7cd56 --- /dev/null +++ b/.changeset/ws-tools-module.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": patch +--- + +Add `ws:tools` so isolate JavaScript can call the agent's own tools; see [`ws:tools` documentation](https://github.com/cloudflare/computer/blob/main/docs/17_isolate_javascript.md#wstools). diff --git a/docs/04_filesystem_interface.md b/docs/04_filesystem_interface.md index 4563fc16..04640a86 100644 --- a/docs/04_filesystem_interface.md +++ b/docs/04_filesystem_interface.md @@ -336,15 +336,19 @@ matched against each candidate's path **relative to `directory`**, not its absolute path — so `**/*.ts` under `/workspace/src` matches `a/b.ts`, not `/workspace/src/a/b.ts`. -The glob supports `*`, `**`, `**/`, and `?`. Character classes and -brace expansions are matched literally. +The glob supports `*`, `**`, `**/`, `?`, and brace alternatives: +`**/*.{ts,tsx}` matches either extension, and groups nest. A brace with +no comma, or with no closing brace, is matched literally, as in a +shell. Character classes are matched literally. `exclude` takes globs of the same shape, matched against the same relative path. An exclusion is decided before the inclusion glob, so it always wins. When an excluded entry is a directory the walk prunes it: neither the directory nor anything beneath it is read, which is what makes skipping `node_modules` or `.git` cheap rather than merely quiet. -`limit` and `offset` then paginate whatever survives. +An exclusion ending in `/**` also excludes the directory it names, so +`node_modules/**` prunes `node_modules` itself; a file of that name is +kept. `limit` and `offset` then paginate whatever survives. ```ts // Every TypeScript file in the project. @@ -355,7 +359,7 @@ const all = await fs.find("/workspace/notes"); // Skip generated trees without descending into them. const sources = await fs.find("/workspace", "**/*.ts", { - exclude: ["node_modules", "node_modules/**", ".git", ".git/**"], + exclude: ["{node_modules,.git}/**"], }); ``` @@ -424,9 +428,8 @@ paginate matching lines. `exclude` takes globs of the same shape as `find`'s, matched against the same directory-relative path and applied before `include`, so an exclusion always wins. An excluded directory is pruned before its children are queried, so the -subtree costs nothing rather than being read and filtered. As with `find`, name -both the directory and its contents to skip a whole subtree: `node_modules/**` -matches what is below `node_modules`, not `node_modules` itself. +subtree costs nothing rather than being read and filtered. As with `find`, +`node_modules/**` skips the whole subtree, `node_modules` included. `path` may be a directory or a single file. Directory searches return matches in deterministic depth-first discovery order, then line order within each @@ -437,7 +440,7 @@ traversal to prune, so `exclude` does not apply to it. const hits = await fs.grep("TODO", "/workspace/src", { ignoreCase: true, include: "**/*.ts", - exclude: ["node_modules", "node_modules/**"], + exclude: ["node_modules/**"], }); for (const hit of hits) { console.log(`${hit.path}:${hit.line}: ${hit.text}`); @@ -526,7 +529,7 @@ maps to `Workspace.fs`: | `symlink` / `readlink` | `symlink` / `readlink` | Same argument order as Node. Targets are stored verbatim and may dangle. | | `watch` | — | Low-level primitive in `fs/watch.ts` (`createWatcher`, `createWatchAsyncIterable`, `WatchHandle`, `WatchOptions`); not exposed on the `WorkspaceFilesystem` class. | | `open` / `FileHandle` | — | Use streams instead. | -| `glob` | `find` | Limited glob support (`*`, `**`, `**/`, and `?`), plus `exclude` for pruning subtrees. | +| `glob` | `find` | Limited glob support (`*`, `**`, `**/`, `?`, and `{a,b}`), plus `exclude` for pruning subtrees. | | — | `grep` | Not in `node:fs`; literal by default, with optional regular expressions. Shares `find`'s `include`/`exclude` globs. | | — | `find` | Recursive directory walk with an optional glob, relative-rooted. | | — | `ls` | Flat list of file paths under a directory (segment-aware). | diff --git a/docs/09_tool_interface.md b/docs/09_tool_interface.md index 37cd862f..de140170 100644 --- a/docs/09_tool_interface.md +++ b/docs/09_tool_interface.md @@ -8,7 +8,7 @@ Computer ships a ready-made tool set for agents that use a `Workspace`, once for | [pi](https://github.com/earendil-works/pi) (`@earendil-works/pi-ai`) | `@cloudflare/computer/tools/pi-ai` | `createPiTools` | | [TanStack AI](https://tanstack.com/ai) (`@tanstack/ai`) | `@cloudflare/computer/tools/tanstack-ai` | `createTanStackTools` | -All three take the same options and build the same tools, with the same names, descriptions, schemas, and limits. Only the shape they return differs. Each entry point imports only `zod` and its own library's types, so a pi agent never loads `ai` and an AI SDK agent never loads pi. The individual AI SDK `create*Tool` functions and `WorkspaceFileStore` come from `@cloudflare/computer/tools`. +All three take the same options and build the same tools, with the same names, descriptions, schemas, and limits. Only the shape they return differs. Each entry point imports only `zod` and its own library's types, so a pi agent never loads `ai` and an AI SDK agent never loads pi. The individual AI SDK `create*Tool` functions and `WorkspaceFileStore` come from `@cloudflare/computer/tools`. `defineExec`, the `exec` tool with no agent library attached, comes from `@cloudflare/computer/tools/core`, which imports only `zod`. The tools wrap three Workspace surfaces: @@ -27,12 +27,13 @@ The tools wrap three Workspace surfaces: | `createWriteTool` | Write a whole file with a UTF-8 byte cap. | | `createEditTool` | Apply atomic targeted replacements and return a unified diff. | | `createListTool` | Page through one directory with file metadata. | -| `createFindTool` | Find paths with `*`, `**`, and `?` globs. | +| `createFindTool` | Find paths with `*`, `**`, `?`, and `{a,b}` globs. | | `createGrepTool` | Search text with regular expressions or fixed strings. | | `createDeleteTool` | Delete a file or directory. | | `createExecTool` | Run a command through a configured Workspace backend. | | `createPublishTool` | Publish a workspace file through `workspace.assets`. | | `WorkspaceFileStore` | Adapt `workspace.fs` to the store used by file tools. | +| `defineExec` | Build the `exec` tool's description, input schema, argument repair, and executor, for a library with no adapter here. | Every tool set names its tools `read`, `ls`, `find`, `grep`, `write`, `edit`, and `delete`. `exec` appears when the Workspace has a backend, unless you pass `exec: {}`. `publish` appears when assets are configured. In read-only mode the set is `read`, `ls`, `find`, and `grep`. @@ -107,21 +108,22 @@ messages.push(message); for (const block of message.content) { if (block.type !== "toolCall") continue; - const { content, isError } = await execute(block); + const { content, isError, details } = await execute(block); messages.push({ role: "toolResult", toolCallId: block.id, toolName: block.name, content, + details, isError, timestamp: Date.now(), }); } ``` -`execute` checks the call's arguments against the tool's schema and returns pi `toolResult` content. A bad call or a failed tool comes back as `isError: true`, so the model can retry and the loop does not throw. pi describes tool parameters with TypeBox, which also accepts plain JSON Schema, so the Zod schemas are converted to JSON Schema and pi needs nothing else. A field with a default stays optional for the model. +`execute` checks the call's arguments against the tool's schema and returns pi `toolResult` content. A bad call or a failed tool comes back as `isError: true`, so the model can retry and the loop does not throw. `details` carries the tool's own output, such as `exec`'s exit code and streams, for the caller and its interface; it is absent when the call failed validation or threw, and for a `read` that returns an image, whose bytes are already in `content`. pi describes tool parameters with TypeBox, which also accepts plain JSON Schema, so the Zod schemas are converted to JSON Schema and pi needs nothing else. A field with a default stays optional for the model. -`read`, `write`, and `edit` carry byte offsets and long verbatim strings, so they ask for pi's `constrainedSampling`. A provider that supports it enforces the schema while sampling, and a malformed `edit` never reaches the tool. The declarations stay open. pi closes a schema itself when the provider supports strict mode, making every field required and the optional ones nullable. `execute` drops a null on an optional field that does not accept one, and keeps a null the tool accepts, such as `exec`'s `input`. +`read`, `write`, and `edit` carry byte offsets and long verbatim strings, so they ask for pi's `constrainedSampling`. A provider that supports it enforces the schema while sampling, and a malformed `edit` never reaches the tool. The declarations stay open. pi closes a schema itself when the provider supports strict mode, making every field required and the optional ones nullable. `execute` drops a null on an optional field that does not accept one. It also parses an `exec` `input` object sent as a string of JSON before validating it. The default is `"prefer"`, which falls back to ordinary tool calling on a provider that cannot enforce a schema. `"require"` fails the request instead, for a pinned model known to support it. `false` turns it off and keeps the schemas open: @@ -169,6 +171,8 @@ createAITools({ write?, edit?, exec?, + execOutput?, + root?, }); ``` @@ -176,15 +180,27 @@ createAITools({ | --- | --- | --- | | `workspace` | required | A `Workspace` or structural equivalent. | | `readonly` | `false` | Omit `write`, `edit`, `delete`, `exec`, and `publish`. Search remains available. | +| `root` | none | Confine the file tools and `publish` to one directory. See [Root](#root). | | `assets` | `true` | Set to `false` to omit `publish`. | | `read` | default caps | Options passed to `createReadTool`. | | `write` | default caps | Options passed to `createWriteTool`. | | `edit` | default caps | Options passed to `createEditTool`. | | `exec` | every backend | An `ExecBackends` map from backend id to `ExecBackendOptions` (`{ description? }`). `{}` omits `exec`. Both types are exported from `@cloudflare/computer/tools`. | +| `execOutput` | 2000 lines, 64 KiB | `{ maxLines?, maxBytes? }`: how much of each `exec` stream the model sees. See [`exec`](#exec). | | `shell` | omitted | Deprecated, and ignored when `exec` is given. `{ backends }` becomes `exec: backends`; `defaultBackend` is ignored. | `createPiTools` and `createTanStackTools` take the same options, plus their own listed above. +### Root + +`root` confines `read`, `write`, `edit`, `delete`, `ls`, `find`, `grep`, and `publish` to one directory: + +```ts +createAITools({ workspace, root: "/workspace" }); +``` + +A relative path resolves against the root, and `.` and `..` are resolved before the check. A path that leaves the root is refused, and so is a path with a symbolic link at or below the root when the filesystem has `lstat`, as `workspace.fs` does. A refusal comes back as the tool's ordinary error result and touches nothing. `exec` is not affected: a backend confines itself, through its own `root`. + ## `read` ```ts @@ -259,15 +275,15 @@ Entries are in name order. A non-final page includes `nextOffset`; pass it as th { path?: string; // default /workspace pattern: string; - exclude?: string[]; + exclude?: string | string[]; limit?: number; // default 200, maximum 1000 offset?: number; } ``` -The pattern is relative to `path`. `*` stays within one path segment, `**` crosses directories, and `?` matches one non-separator character. Results contain `path` and `type`; a non-final page includes `nextOffset`. Pagination reaches `workspace.fs.find`, which walks directory children in fixed-size pages and stops after collecting the requested page instead of materializing every match. +The pattern is relative to `path`. `*` stays within one path segment, `**` crosses directories, `?` matches one non-separator character, and `{a,b}` matches either alternative. Results contain `path` and `type`; a non-final page includes `nextOffset`. Pagination reaches `workspace.fs.find`, which walks directory children in fixed-size pages and stops after collecting the requested page instead of materializing every match. -`exclude` takes globs of the same shape, matched against the same relative path, and beats the inclusion pattern. An excluded directory is pruned rather than filtered, so `exclude: ["node_modules", "node_modules/**"]` keeps the walk out of a package tree instead of walking it and discarding the results. +`exclude` takes one glob or a list, of the same shape, matched against the same relative path, and beats the inclusion pattern. An excluded directory is pruned rather than filtered, so `exclude: "node_modules/**"` keeps the walk out of a package tree instead of walking it and discarding the results: a glob ending in `/**` also excludes the directory it names. ## `grep` @@ -275,8 +291,8 @@ The pattern is relative to `path`. `*` stays within one path segment, `**` cross { path?: string; // default /workspace query: string; - include?: string; // glob relative to path - exclude?: string[]; // globs pruned from the walk + include?: string | string[]; // globs relative to path + exclude?: string | string[]; // globs pruned from the walk regex?: boolean; // default false ignoreCase?: boolean; // default false context?: number; // 0 through 10 @@ -287,7 +303,7 @@ The pattern is relative to `path`. `*` stays within one path segment, `**` cross The AI tool defaults to literal, case-sensitive matching. Set `regex: true` to interpret `query` as a regular expression and `ignoreCase: true` to ignore letter case. Matches include path, line number, text, and optional numbered context. Invalid regular expressions return a structured error. A non-final page includes `nextOffset`. -`exclude` works exactly as it does on `find`: globs of the same shape, matched against the same relative path, applied before `include` so an exclusion always wins. An excluded directory is pruned before its children are queried rather than being read and filtered, so `exclude: ["node_modules", "node_modules/**"]` keeps the search out of a package tree. Name both forms, since `node_modules/**` matches what is below the directory rather than the directory itself. A single-file search has no traversal to prune, so `exclude` does not apply to it. +`exclude` works exactly as it does on `find`: globs of the same shape, matched against the same relative path, applied before `include` so an exclusion always wins. An excluded directory is pruned before its children are queried rather than being read and filtered, so `exclude: "node_modules/**"` keeps the search out of a package tree. Several `include` globs search files matching any of them. A single-file search has no traversal to prune, so `exclude` does not apply to it. The tool passes `include`, `exclude`, `limit`, and `offset` through one `workspace.fs.grep` call. The storage search pages matching files and stops after the requested matches, so an included search does not build the full file or match list in the tool layer. Directory searches return matches in deterministic depth-first discovery order, then line order within each file. They are not globally sorted by full path. @@ -304,7 +320,7 @@ The schema is `{ path, content }`. Writing overwrites the file and preserves its ## `edit` ```ts -createEditTool({ store, maxBytes? }); // default 2 MiB +createEditTool({ store, maxBytes?, maxDiffLines?, maxDiffBytes? }); // defaults 2 MiB, 2000, 128 KiB ``` The schema is: @@ -320,6 +336,8 @@ Every `oldText` must identify one unique, non-overlapping range in the original The tool applies the batch atomically, preserves the byte order mark, line ending style, and file mode, and returns a unified patch plus `firstChangedLine`. +Diffing costs roughly the square of the number of changed lines, so the diff is bounded while the edit is not. When the replaced text, counted in lines on either side of each edit, exceeds `maxDiffLines`, the diff and patch are empty. Otherwise each is cut to `maxDiffBytes` on a character boundary. Either way the result sets `diffTruncated: true`. + `edit`, `write`, and `delete` share locks through the store's stable `lockIdentity`. Every `WorkspaceFileStore` over the same `workspace.fs` uses the same identity, including adapters created by separate `createAITools()` calls. A write cannot land between edit's read and write phases, while unrelated workspaces and paths remain independent. Recursive deletion also locks the whole subtree, so mutations to ancestors or descendants cannot interleave with it. ## `delete` @@ -344,12 +362,14 @@ The tool offers only the arguments that can work: | Backends | Arguments | | --- | --- | | One shell backend | `command`, `cwd`, `env` | -| One callable backend | `command`, `cwd`, `env`, `input` | +| One callable backend | `command`, `cwd`, `env`, `input` (an object) | | More than one | `command`, `cwd`, `backend` (required), `env`, plus `input` when any is callable | A `backend` value the model sends anyway is dropped when only one backend is configured. The output still names the backend that ran. -Long output follows pi's bash tool. Each of stdout and stderr shows its last `maxLines` lines (2000) or `maxBytes` (64 KiB), whichever is hit first. The tool passes the same limits to the runtime, which saves the full output to a Workspace file (see [Long output](./05_runtime_interface.md#long-output)), and the reply ends with a note naming it: +`input` is an object of JSON values, described to the model as an input object. Offered any JSON value, some models send the object they mean as a string of JSON. A string that parses to an object is replaced by that object before validation, through the definition's `prepareArguments`; `createPiTools` applies it, and a caller of `defineExec` should too. + +Long output follows pi's bash tool. Each of stdout and stderr shows its last `maxLines` lines (2000) or `maxBytes` (64 KiB), whichever is hit first. Set them with `execOutput` on a tool set, or on `createExecTool` directly. The tool passes the same limits to the runtime, which saves the full output to a Workspace file (see [Long output](./05_runtime_interface.md#long-output)), and the reply ends with a note naming it: ```text line 2999 @@ -400,7 +420,7 @@ interface MutableFileStore extends FileStore { ## Conventions for agents -- Tools take absolute paths. Resolve user input against the configured workspace root before calling them. See [01. VFS](./01_vfs.md). +- Tools take absolute paths. Pass `root` to resolve relative paths against a directory and refuse paths outside it, or resolve user input yourself before calling them. See [01. VFS](./01_vfs.md). - The `read` tool returns line and byte continuation offsets. Pass both back on the next call instead of asking for the whole file again. - Tell the model that each `edit` batch applies against the original file content. Treating each edit as an incremental change can produce overlapping edits, which the tool rejects. - Describe every shell backend in plain language. The model reads these descriptions when deciding where to run a command. diff --git a/docs/17_isolate_javascript.md b/docs/17_isolate_javascript.md index bfae193e..a7504b69 100644 --- a/docs/17_isolate_javascript.md +++ b/docs/17_isolate_javascript.md @@ -152,8 +152,10 @@ Caller source can import four kinds of module, and all of them are fixed when th ```ts import { createArtifactsModule } from "@cloudflare/computer/modules/artifacts"; +import { createAssetsModule } from "@cloudflare/computer/modules/assets"; import { createContainerModule } from "@cloudflare/computer/modules/container"; import { createGitModule } from "@cloudflare/computer/modules/git"; +import { createToolsModule } from "@cloudflare/computer/modules/tools"; new WorkerJavaScriptBackend({ loader: env.LOADER, @@ -161,7 +163,9 @@ new WorkerJavaScriptBackend({ "tar-stream": TAR_STREAM_BUNDLE, "ws:git": createGitModule(), "ws:artifacts": createArtifactsModule(), + "ws:assets": createAssetsModule(), "ws:container": createContainerModule(), + "ws:tools": createToolsModule(() => agentTools, { exclude: ["exec"] }), "ws:weather": { forecast: ([city]) => lookUpForecast(String(city)), }, @@ -191,7 +195,7 @@ Modules code can import: - Node.js built-ins: `node:path`, `node:url`, ... Bare names such as `path` work too. ``` -A factory adds its own text through a `description` property, as the prebuilt modules do. An object of functions is listed by its export names; say more about it in the `exec` tool's backend description if the model needs it. +A factory adds its own text through a `description` property, as the prebuilt modules do. The backend reads it each time it describes itself, not once when it is constructed, so a factory can define `description` as a getter and describe something that exists only later, as `ws:tools` does with the agent's tools. An object of functions is listed by its export names; say more about it in the `exec` tool's backend description if the model needs it. ### Built-in filesystem @@ -229,7 +233,7 @@ modules: { } ``` -When the functions need the Workspace's Git client, Artifacts client, or runtime, pass a factory instead. The backend calls it once when it connects to its Workspace. This is how the prebuilt modules work: +When the functions need the Workspace's Git client, Artifacts client, runtime, or assets client, pass a factory instead. `host.assets` is absent when the Workspace has no assets configured. The backend calls it once when it connects to its Workspace. This is how the prebuilt modules work: ```ts modules: { @@ -271,11 +275,47 @@ import { clone, diff, status, log, cli } from "ws:git"; ### `ws:artifacts` ```js -import { create, get, list, importArtifact, deleteArtifact } from "ws:artifacts"; +import { create, get, list, importArtifact, deleteArtifact, createToken, share } from "ws:artifacts"; ``` `createArtifactsModule()` from `@cloudflare/computer/modules/artifacts` wraps the Workspace's Artifacts client. Calls that change Artifacts need a read-write backend. `importArtifact()` fetches from a caller-chosen URL on the host, so it is denied unless you pass `createArtifactsModule({ allowNetwork: true })`. Every call fails clearly when no Artifacts binding is configured. +Two functions mint the git token that cloning or pushing a repository needs: + +- `createToken(name, scope?, ttl?)` returns `{ id, plaintext, scope, expiresAt }`. +- `share(name, { scope?, ttl? })` returns the repository's remote URL with a token embedded, the same URL `artifacts share` prints. It resolves the repository first, so a missing one fails without minting a token. + +`scope` is `"read"` (the default) or `"write"`, and a write token needs a read-write backend. `ttl` is seconds, or a duration such as `"15m"` or `"2h30m"`; without it the binding's default applies. The URL from `share` is a credential: pass it to `ws:container` through `env` rather than in the command text, and do not return it from the run. + +```js +import { share } from "ws:artifacts"; +import { exec } from "ws:container"; + +export default async function () { + const url = await share("site", { scope: "write", ttl: "30m" }); + const push = await exec('git push "$REMOTE" HEAD:main', { cwd: "/workspace/site", env: { REMOTE: url } }); + return { pushed: push.exitCode === 0 }; +} +``` + +### `ws:assets` + +```js +import { publish } from "ws:assets"; +``` + +`createAssetsModule()` from `@cloudflare/computer/modules/assets` publishes a Workspace file through the Workspace's assets client (see [14. Assets interface](./14_assets_interface.md)) and returns a time-limited URL: + +```js +import { publish } from "ws:assets"; + +export default async function () { + return publish("/workspace/out/report.pdf", { expiresAfter: "1d", disposition: "attachment" }); +} +``` + +`publish(path, { expiresAfter?, filename?, disposition?, contentType? })` resolves `path` against the backend root like any other host call. `expiresAfter` is milliseconds or a duration such as `"30m"`; the default is one hour, `createAssetsModule({ defaultExpiresAfterMs })` changes it, and the assets client caps it at seven days. `disposition` is `"inline"` or `"attachment"`. If the Workspace has no assets client, the JavaScript backend fails to connect. + ### `ws:container` `createContainerModule()` from `@cloudflare/computer/modules/container` lets JavaScript run shell commands in the Workspace's container backend. With it, JavaScript is the only backend the model sees, and the container is something that JavaScript can call: @@ -326,6 +366,57 @@ A few limits follow from `exec` being a host call: A container command can write to the Workspace, so `exec` refuses to run on a read-only backend. Whether it can reach the network follows `ContainerBackend`'s own `egress` setting, not the JavaScript backend's. +Two options shape every command: + +| Option | Default | Notes | +| --- | --- | --- | +| `prelude` | none | Shell text run before each command, joined to it by a newline, such as `set -o pipefail` or `export` lines. A newline keeps a command that opens with a comment or a shebang intact, and does not skip the command when the prelude's last status is non-zero. | +| `maxOutputLines` | the runtime's 2000 | The most lines of each stream the result keeps, alongside `maxOutputBytes`. | + +```ts +createContainerModule({ + prelude: ["set -o pipefail", 'export GIT_AUTHOR_NAME="${GIT_AUTHOR_NAME:-Agent}"'].join("\n"), + maxOutputLines: 200, +}); +``` + +### `ws:tools` + +`createToolsModule(tools, { exclude? })` from `@cloudflare/computer/modules/tools` lets code call the agent's own tools, so a search or a fetch repeated many times costs one run instead of a model turn each: + +```js +import { grep } from "ws:tools"; + +export default async function ({ names }) { + const hits = {}; + for (const name of names) hits[name] = await grep({ query: name, path: "/workspace/src" }); + return hits; +} +``` + +`tools` is a function returning `{ name, execute(args, { signal }) }` objects, read when the backend connects, so tools built after the Workspace can be offered. Each tool becomes an export under its own name, which takes one object of the tool's arguments and returns what `execute` returns. `exclude` leaves tools out; leave out `exec` itself so a run cannot start runs. Names that cannot be a module export, such as `web-fetch`, are left out too. The module's description lists the current tool names. + +The module checks only that each call passes one object. Validate arguments against the tool's own schema in `execute`, and return JSON-compatible data, draining a streaming tool to its final value. `createPiTools`' `execute` already validates and drains, so a pi agent can pass its tools through directly: + +```ts +const pi = createPiTools({ workspace }); + +createToolsModule( + () => + pi.tools.map((tool) => ({ + name: tool.name, + execute: async (args, { signal }) => { + const result = await pi.execute( + { id: crypto.randomUUID(), name: tool.name, arguments: args }, + { abortSignal: signal }, + ); + return { details: result.details ?? null, isError: result.isError }; + }, + })), + { exclude: ["exec"] }, +); +``` + ## Isolation and lifecycle Each execution receives a fresh Dynamic Worker with: diff --git a/docs/README.md b/docs/README.md index e7199b6b..679e3051 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,7 +20,7 @@ It provides: - R2-backed mounts for pre-filling read-only data into the workspace tree. - Durability over DO restarts for all file operations. - Pluggable execution backends selected through `workspace.runtime`: a Cloudflare Container shell, a just-bash Dynamic Worker, or an isolated ECMAScript-module Dynamic Worker. - - Isolated JavaScript with structured input/results, durable relative imports, configured libraries, durable `node:fs/promises`, host modules such as `ws:git` and `ws:container`, and managed execution records. + - Isolated JavaScript with structured input/results, durable relative imports, configured libraries, durable `node:fs/promises`, host modules such as `ws:git`, `ws:container`, `ws:assets` and `ws:tools`, and managed execution records. - Workspace constructable without a backend, for filesystem-only use cases. - Out-of-the-box agent tools for the AI SDK (`createAITools()` in `@cloudflare/computer/tools/ai-sdk`), pi (`createPiTools()` in `@cloudflare/computer/tools/pi-ai`), and TanStack AI (`createTanStackTools()` in `@cloudflare/computer/tools/tanstack-ai`). @@ -51,8 +51,11 @@ The package ships several entrypoints: | `@cloudflare/computer/artifacts` | `createArtifact`, an optionally session-scoped wrapper over the Cloudflare Artifacts Workers binding, plus its argv CLI. | | `@cloudflare/computer/modules/container` | `createContainerModule()` for `ws:container`: run container commands from isolate JavaScript. | | `@cloudflare/computer/modules/git` | `createGitModule()` for `ws:git`: confined Git from isolate JavaScript. | -| `@cloudflare/computer/modules/artifacts` | `createArtifactsModule()` for `ws:artifacts`: Artifacts from isolate JavaScript. | +| `@cloudflare/computer/modules/artifacts` | `createArtifactsModule()` for `ws:artifacts`: Artifacts and git tokens from isolate JavaScript. | +| `@cloudflare/computer/modules/assets` | `createAssetsModule()` for `ws:assets`: publish a Workspace file from isolate JavaScript. | +| `@cloudflare/computer/modules/tools` | `createToolsModule()` for `ws:tools`: call the agent's own tools from isolate JavaScript. | | `@cloudflare/computer/tools` | AI SDK tools for agents: read, write, edit, ls, optional exec, and optional publish. | +| `@cloudflare/computer/tools/core` | `defineExec()`: the `exec` tool's description, schema, and executor, with no agent library. | | `@cloudflare/computer/tools/ai-sdk` | `createAITools()`: the AI SDK tool set for a Workspace. | | `@cloudflare/computer/tools/pi-ai` | `createPiTools()`: the same tool set for pi, as declarations plus a function that runs a tool call. | | `@cloudflare/computer/tools/tanstack-ai` | `createTanStackTools()`: the same tool set for TanStack AI, as the list `chat({ tools })` takes. | diff --git a/packages/computer/README.md b/packages/computer/README.md index 4e52b6ee..f012d80c 100644 --- a/packages/computer/README.md +++ b/packages/computer/README.md @@ -435,8 +435,11 @@ on a computerd instance. | `@cloudflare/computer/backends/worker-javascript` | `WorkerJavaScriptBackend`, configured libraries, durable imports, `node:fs/promises`, and host modules. | | `@cloudflare/computer/modules/container` | `createContainerModule()` for `ws:container`: run container commands from isolate JavaScript. | | `@cloudflare/computer/modules/git` | `createGitModule()` for `ws:git`: confined Git from isolate JavaScript. | -| `@cloudflare/computer/modules/artifacts` | `createArtifactsModule()` for `ws:artifacts`: Artifacts from isolate JavaScript. | +| `@cloudflare/computer/modules/artifacts` | `createArtifactsModule()` for `ws:artifacts`: Artifacts and git tokens from isolate JavaScript. | +| `@cloudflare/computer/modules/assets` | `createAssetsModule()` for `ws:assets`: publish a Workspace file from isolate JavaScript. | +| `@cloudflare/computer/modules/tools` | `createToolsModule()` for `ws:tools`: call the agent's own tools from isolate JavaScript. | | `@cloudflare/computer/tools` | AI SDK tools for agents: `read`, `ls`, `find`, `grep`, `write`, `edit`, `delete`, and optional `exec` and `publish`. | +| `@cloudflare/computer/tools/core` | `defineExec()`: the `exec` tool's description, schema, and executor, with no agent library. | | `@cloudflare/computer/tools/ai-sdk` | `createAITools()`: the AI SDK tool set for a Workspace. | | `@cloudflare/computer/tools/pi-ai` | `createPiTools()`: the same tool set for pi (`@earendil-works/pi-ai`). | | `@cloudflare/computer/tools/tanstack-ai` | `createTanStackTools()`: the same tool set for TanStack AI (`@tanstack/ai`). | diff --git a/packages/computer/package.json b/packages/computer/package.json index 6ef5b251..d1d2e182 100644 --- a/packages/computer/package.json +++ b/packages/computer/package.json @@ -43,10 +43,22 @@ "types": "./dist/modules/artifacts.d.ts", "import": "./dist/modules/artifacts.js" }, + "./modules/assets": { + "types": "./dist/modules/assets.d.ts", + "import": "./dist/modules/assets.js" + }, + "./modules/tools": { + "types": "./dist/modules/tools.d.ts", + "import": "./dist/modules/tools.js" + }, "./tools": { "types": "./dist/tools/index.d.ts", "import": "./dist/tools/index.js" }, + "./tools/core": { + "types": "./dist/tools/core.d.ts", + "import": "./dist/tools/core.js" + }, "./tools/pi-ai": { "types": "./dist/tools/pi-ai.d.ts", "import": "./dist/tools/pi-ai.js" diff --git a/packages/computer/rolldown.config.ts b/packages/computer/rolldown.config.ts index 22b9a8a2..b3a94725 100644 --- a/packages/computer/rolldown.config.ts +++ b/packages/computer/rolldown.config.ts @@ -32,11 +32,14 @@ export default defineConfig({ "assets/index": "src/assets/index.ts", "tools/index": "src/tools/index.ts", "tools/ai-sdk": "src/tools/ai-sdk/index.ts", + "tools/core": "src/tools/core.ts", "tools/pi-ai": "src/tools/pi-ai/index.ts", "tools/tanstack-ai": "src/tools/tanstack-ai/index.ts", "modules/container": "src/modules/container.ts", "modules/git": "src/modules/git.ts", "modules/artifacts": "src/modules/artifacts.ts", + "modules/assets": "src/modules/assets.ts", + "modules/tools": "src/modules/tools.ts", "backends/container-legacy/index": "src/backends/container-legacy/index.ts", "backends/container/index": "src/backends/container/index.ts", "backends/worker-javascript/index": "src/backends/worker-javascript/index.ts", diff --git a/packages/computer/src/backends/worker-javascript/module-graph.ts b/packages/computer/src/backends/worker-javascript/module-graph.ts index 98edd9c2..9ffeea4e 100644 --- a/packages/computer/src/backends/worker-javascript/module-graph.ts +++ b/packages/computer/src/backends/worker-javascript/module-graph.ts @@ -105,7 +105,7 @@ const FILESYSTEM_DESCRIPTION = export function parseModules(modules: Readonly>): ParsedModules { const source: Record = Object.create(null); const host = new Map(); - const lines = [FILESYSTEM_DESCRIPTION]; + const lines: Array string)> = [FILESYSTEM_DESCRIPTION]; for (const [specifier, module] of Object.entries(modules)) { const name = `\`${specifier}\``; if (BUILT_IN_MODULES.some((builtIn) => builtIn === specifier)) { @@ -131,7 +131,7 @@ export function parseModules(modules: Readonly>) } if (typeof module === "function") { host.set(specifier, module); - lines.push(`- ${name}: ${module.description ?? "a host module."}`); + lines.push(() => `- ${name}: ${module.description ?? "a host module."}`); continue; } if (module === null || typeof module !== "object" || Array.isArray(module)) { @@ -144,7 +144,13 @@ export function parseModules(modules: Readonly>) const exports = Object.keys(module).map((key) => `\`${key}\``); lines.push(`- ${name}: exports ${exports.join(", ")}.`); } - return { source, host, description: lines.join("\n") }; + return { + source, + host, + get description() { + return lines.map((line) => (typeof line === "string" ? line : line())).join("\n"); + }, + }; } /** diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts index a7d95e23..a9c7e8af 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.test.ts @@ -1074,6 +1074,21 @@ describe("WorkerJavaScriptBackend", () => { ); }); + it("reads a factory's description each time it describes itself", () => { + let names = ["read"]; + const factory = Object.defineProperty(() => ({ run: () => null }), "description", { + get: () => `Tools: ${names.join(", ")}.`, + }); + const backend = new WorkerJavaScriptBackend({ + loader: throwingLoader("must not load"), + modules: { "ws:tools": factory }, + }); + + expect(backend.description).toContain("- `ws:tools`: Tools: read."); + names = ["read", "grep"]; + expect(backend.description).toContain("- `ws:tools`: Tools: read, grep."); + }); + it("builds host modules from the Workspace services when it connects", async () => { const db = new Database(new SQLiteTestStorage()); initializeSchema(db, () => 0); @@ -1098,6 +1113,32 @@ describe("WorkerJavaScriptBackend", () => { expect(seen).toBe(git); }); + it("hands host modules the Workspace's assets client", async () => { + const db = new Database(new SQLiteTestStorage()); + initializeSchema(db, () => 0); + const assets = { marker: "assets" }; + const seen: unknown[] = []; + const backend = new WorkerJavaScriptBackend({ + loader: throwingLoader("must not load"), + modules: { + "ws:test": (host) => { + seen.push(host.assets); + return { run: async () => null }; + }, + }, + }); + const host = { + db, + fs: new WorkspaceFilesystem(db), + git: undefined as never, + artifacts: undefined as never, + runtime: undefined as never, + }; + await backend.connect({ ...host, assets: assets as never }); + await backend.connect(host); + expect(seen).toEqual([assets, undefined]); + }); + it("does not dispatch inherited members of a host module", async () => { const db = new Database(new SQLiteTestStorage()); initializeSchema(db, () => 0); diff --git a/packages/computer/src/backends/worker-javascript/worker-javascript.ts b/packages/computer/src/backends/worker-javascript/worker-javascript.ts index 0f4b9624..4f41085c 100644 --- a/packages/computer/src/backends/worker-javascript/worker-javascript.ts +++ b/packages/computer/src/backends/worker-javascript/worker-javascript.ts @@ -157,10 +157,9 @@ export class WorkerJavaScriptBackend implements WorkspaceModuleBackend { readonly protocol = "module" as const; readonly type = "worker-javascript"; readonly callable = true; - /** What this backend tells a model: the source language and every importable module. */ - readonly description: string; readonly id: string; readonly #options: ResolvedWorkerJavaScriptBackendOptions; + readonly #header: string; constructor(options: WorkerJavaScriptBackendOptions) { this.id = options.id ?? "worker-javascript"; @@ -241,13 +240,20 @@ export class WorkerJavaScriptBackend implements WorkspaceModuleBackend { compatibilityDate, compatibilityFlags: options.compatibilityFlags ?? ["nodejs_compat"], }; - this.description = [ + this.#header = [ "`command` is ECMAScript module source, run in an isolated JavaScript runtime. Relative imports resolve from `cwd` in the workspace.", "Put the work in `export default async function (input) { ... }` and call `node:fs` and the other modules below inside it, since the module's top level can't do I/O. To run a file you've already written, re-export it: `export { default } from \"./main.js\"`.", ...(resolvedEgress.mode === "none" ? ["Code has no direct network access."] : []), ...(this.#options.access === "read" ? ["The workspace is read-only here."] : []), "", "Modules code can import:", + ].join("\n"); + } + + /** What this backend tells a model: the source language and every importable module. */ + get description(): string { + return [ + this.#header, this.#options.modules.description, ...(hasNodeModules(this.#options.compatibilityFlags) ? [NODE_MODULES_DESCRIPTION] : []), ].join("\n"); @@ -275,7 +281,12 @@ class JavaScriptBackendHandle implements WorkspaceModuleBackendHandle { this.#host = host; const functions = new Map(); for (const [specifier, factory] of options.modules.host) { - const built = factory({ git: host.git, artifacts: host.artifacts, runtime: host.runtime }); + const built = factory({ + git: host.git, + artifacts: host.artifacts, + runtime: host.runtime, + ...(host.assets === undefined ? {} : { assets: host.assets }), + }); assertHostModuleExports(specifier, built); functions.set(specifier, built); } diff --git a/packages/computer/src/modules/artifacts.test.ts b/packages/computer/src/modules/artifacts.test.ts new file mode 100644 index 00000000..90fd8ab3 --- /dev/null +++ b/packages/computer/src/modules/artifacts.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from "vitest"; + +import type { WorkspaceModuleCallContext, WorkspaceModuleHost } from "../runtime/types.js"; +import { createArtifactsModule } from "./artifacts.js"; + +function fakeArtifacts() { + const tokens: Array<{ name: string; scope: string | undefined; ttl: number | undefined }> = []; + const gets: string[] = []; + return { + tokens, + gets, + client: { + async get(name: string) { + gets.push(name); + if (name === "missing") throw new Error("repo not found"); + return { name, remote: `https://artifacts.example/git/${name}.git` }; + }, + async createToken(name: string, scope?: string, ttl?: number) { + tokens.push({ name, scope, ttl }); + return { + id: "tok_1", + plaintext: "secret/value?expires=123", + scope: scope ?? "read", + expiresAt: "2030-01-01T00:00:00Z", + }; + }, + }, + }; +} + +function build(artifacts: ReturnType["client"]) { + // SAFETY: The functions under test only call get and createToken. + const host = { artifacts, git: undefined, runtime: undefined } as unknown as WorkspaceModuleHost; + const functions = createArtifactsModule()(host); + const { createToken, share } = functions; + if (!createToken || !share) throw new Error("ws:artifacts must export createToken and share"); + return { createToken, share }; +} + +function callContext(access: "read" | "read-write" = "read-write"): WorkspaceModuleCallContext { + return { + signal: new AbortController().signal, + deadline: Date.now() + 60_000, + access, + resolvePath: async (path) => path, + }; +} + +describe("createArtifactsModule createToken", () => { + it("mints a read token by default", async () => { + const { client, tokens } = fakeArtifacts(); + const { createToken } = build(client); + + await expect(createToken(["repo"], callContext("read"))).resolves.toEqual({ + id: "tok_1", + plaintext: "secret/value?expires=123", + scope: "read", + expiresAt: "2030-01-01T00:00:00Z", + }); + expect(tokens).toEqual([{ name: "repo", scope: "read", ttl: undefined }]); + }); + + it("takes a ttl in seconds or as a duration", async () => { + const { client, tokens } = fakeArtifacts(); + const { createToken } = build(client); + + await createToken(["repo", "write", 60], callContext()); + await createToken(["repo", null, "15m"], callContext()); + expect(tokens.map(({ scope, ttl }) => ({ scope, ttl }))).toEqual([ + { scope: "write", ttl: 60 }, + { scope: "read", ttl: 900 }, + ]); + }); + + it("refuses a write token without write access", async () => { + const { client, tokens } = fakeArtifacts(); + const { createToken } = build(client); + + await expect(createToken(["repo", "write"], callContext("read"))).rejects.toThrow( + /write access/, + ); + expect(tokens).toEqual([]); + }); + + it.each([ + [[], /name must be a non-empty string/], + [["repo", "admin"], /scope must be "read" or "write"/], + [["repo", "read", 1.5], /positive whole number/], + [["repo", "read", "soon"], /not a duration/], + [["repo", "read", true], /number of seconds or a duration/], + ])("rejects %j", async (args, message) => { + const { client, tokens } = fakeArtifacts(); + const { createToken } = build(client); + + await expect(createToken(args as never, callContext())).rejects.toThrow(message); + expect(tokens).toEqual([]); + }); +}); + +describe("createArtifactsModule share", () => { + it("returns the repo's remote with a token embedded", async () => { + const { client, tokens } = fakeArtifacts(); + const { share } = build(client); + + await expect(share(["repo"], callContext("read"))).resolves.toBe( + "https://x:secret%2Fvalue@artifacts.example/git/repo.git", + ); + expect(tokens).toEqual([{ name: "repo", scope: "read", ttl: undefined }]); + }); + + it("passes the scope and ttl through", async () => { + const { client, tokens } = fakeArtifacts(); + const { share } = build(client); + + await share(["repo", { scope: "write", ttl: "2h" }], callContext()); + expect(tokens).toEqual([{ name: "repo", scope: "write", ttl: 7200 }]); + }); + + it("does not mint a token for a missing repo", async () => { + const { client, tokens } = fakeArtifacts(); + const { share } = build(client); + + await expect(share(["missing"], callContext())).rejects.toThrow(/not found/); + expect(tokens).toEqual([]); + }); + + it("refuses a write URL without write access", async () => { + const { client, gets } = fakeArtifacts(); + const { share } = build(client); + + await expect(share(["repo", { scope: "write" }], callContext("read"))).rejects.toThrow( + /write access/, + ); + expect(gets).toEqual([]); + }); + + it("rejects options that are not an object", async () => { + const { client } = fakeArtifacts(); + const { share } = build(client); + + await expect(share(["repo", "write"], callContext())).rejects.toThrow(/must be an object/); + }); + + it("describes both", () => { + const description = createArtifactsModule().description ?? ""; + expect(description).toContain("createToken(name"); + expect(description).toContain("share(name"); + }); +}); diff --git a/packages/computer/src/modules/artifacts.ts b/packages/computer/src/modules/artifacts.ts index e577c5ef..c65abf39 100644 --- a/packages/computer/src/modules/artifacts.ts +++ b/packages/computer/src/modules/artifacts.ts @@ -7,12 +7,15 @@ // `allowNetwork: true`: the request runs from the host, so the // isolate's own egress settings do not stop it. -import type { ArtifactClient } from "../artifacts/index.js"; +import { credentialURL } from "../artifacts/cli.js"; +import { parseDuration } from "../artifacts/duration.js"; +import type { ArtifactClient, ArtifactScope } from "../artifacts/index.js"; import type { WorkspaceModuleCallContext, WorkspaceModuleFactory, WorkspaceModuleFunctions, WorkspaceModuleHost, + WorkspaceRuntimeValue, } from "../runtime/types.js"; /** Options for {@link createArtifactsModule}. */ @@ -70,9 +73,29 @@ export function createArtifactsModule( requireWrite(context, "Artifacts delete"); return host.artifacts.delete(String(name)); }, + async createToken([name, scope, ttl], context) { + const tokenScope = parseScope(scope, "createToken"); + if (tokenScope === "write") requireWrite(context, "A write token"); + const token = await host.artifacts.createToken( + requireName(name, "createToken"), + tokenScope, + parseTtl(ttl, "createToken"), + ); + return { ...token }; + }, + async share([name, shareOptions], context) { + const repo = requireName(name, "share"); + const options = optionalObject(shareOptions, "share"); + const tokenScope = parseScope(options.scope, "share"); + if (tokenScope === "write") requireWrite(context, "A write token"); + const ttl = parseTtl(options.ttl, "share"); + const info = await host.artifacts.get(repo); + const token = await host.artifacts.createToken(repo, tokenScope, ttl); + return credentialURL(info.remote, token.plaintext); + }, }); return Object.assign(create, { - description: `Git repositories stored in Cloudflare Artifacts: \`create(name)\`, \`get(name)\`, \`list()\`, \`importArtifact(name, source)\`, and \`deleteArtifact(name)\`.${allowNetwork ? "" : " Importing from a remote URL is not allowed."}`, + description: `Git repositories stored in Cloudflare Artifacts: \`create(name)\`, \`get(name)\`, \`list()\`, \`importArtifact(name, source)\`, and \`deleteArtifact(name)\`.${allowNetwork ? "" : " Importing from a remote URL is not allowed."} \`createToken(name, scope?, ttl?)\` mints a git token ("read" by default). \`share(name, { scope?, ttl? })\` returns a clone-ready URL with a token embedded; it is a credential, so never print or return it. \`ttl\` is seconds or a duration like "15m".`, }); } @@ -81,3 +104,49 @@ function requireWrite(context: WorkspaceModuleCallContext, operation: string) { throw new Error(`${operation} requires Workspace write access.`); } } + +function requireName(value: WorkspaceRuntimeValue | undefined, operation: string): string { + if (typeof value !== "string" || value.length === 0) { + throw new TypeError(`${operation}: name must be a non-empty string.`); + } + return value; +} + +function parseScope(value: WorkspaceRuntimeValue | undefined, operation: string): ArtifactScope { + if (value === undefined || value === null) return "read"; + if (value === "read" || value === "write") return value; + throw new TypeError( + `${operation}: scope must be "read" or "write", got ${JSON.stringify(value)}.`, + ); +} + +function parseTtl(value: WorkspaceRuntimeValue | undefined, operation: string): number | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value === "number") { + if (!Number.isInteger(value) || value <= 0) { + throw new TypeError(`${operation}: ttl must be a positive whole number of seconds.`); + } + return value; + } + if (typeof value === "string") { + try { + return parseDuration(value); + } catch (error) { + throw new TypeError( + `${operation}: ttl ${JSON.stringify(value)} is not a duration like "15m": ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + throw new TypeError(`${operation}: ttl must be a number of seconds or a duration.`); +} + +function optionalObject( + value: WorkspaceRuntimeValue | undefined, + operation: string, +): Record { + if (value === undefined || value === null) return {}; + if (typeof value !== "object" || Array.isArray(value)) { + throw new TypeError(`${operation}: options must be an object.`); + } + return value; +} diff --git a/packages/computer/src/modules/assets.test.ts b/packages/computer/src/modules/assets.test.ts new file mode 100644 index 00000000..030b4581 --- /dev/null +++ b/packages/computer/src/modules/assets.test.ts @@ -0,0 +1,136 @@ +import { describe, expect, it } from "vitest"; + +import type { ShareOptions } from "../assets/index.js"; +import type { WorkspaceModuleCallContext, WorkspaceModuleHost } from "../runtime/types.js"; +import { createAssetsModule } from "./assets.js"; + +function fakeAssets() { + const calls: Array<{ path: string; options: ShareOptions }> = []; + return { + calls, + client: { + async share(path: string, options: ShareOptions) { + calls.push({ path, options }); + return `https://assets.example/${path.split("/").pop()}`; + }, + }, + }; +} + +function build( + assets: ReturnType["client"] | undefined, + options?: Parameters[0], +) { + // SAFETY: The module only reads host.assets. + const host = { + assets, + git: undefined, + artifacts: undefined, + runtime: undefined, + } as unknown as WorkspaceModuleHost; + const publish = createAssetsModule(options)(host).publish; + if (!publish) throw new Error("ws:assets must export publish"); + return publish; +} + +function callContext( + overrides: Partial = {}, +): WorkspaceModuleCallContext { + return { + signal: new AbortController().signal, + deadline: Date.now() + 60_000, + access: "read", + resolvePath: async (path) => (path.startsWith("/") ? path : `/workspace/${path}`), + ...overrides, + }; +} + +describe("createAssetsModule", () => { + it("publishes the resolved path for an hour by default", async () => { + const { client, calls } = fakeAssets(); + const publish = build(client); + + await expect(publish(["out/report.pdf"], callContext())).resolves.toBe( + "https://assets.example/report.pdf", + ); + expect(calls).toEqual([ + { path: "/workspace/out/report.pdf", options: { expiresAfter: 60 * 60 * 1000 } }, + ]); + }); + + it("accepts milliseconds or a duration, and the share options", async () => { + const { client, calls } = fakeAssets(); + const publish = build(client); + + await publish(["/workspace/a.txt", { expiresAfter: 5000 }], callContext()); + await publish( + [ + "/workspace/b.txt", + { + expiresAfter: "2h30m", + filename: "b.csv", + disposition: "attachment", + contentType: "text/csv", + }, + ], + callContext(), + ); + expect(calls.map((call) => call.options)).toEqual([ + { expiresAfter: 5000 }, + { + expiresAfter: 9_000_000, + filename: "b.csv", + disposition: "attachment", + contentType: "text/csv", + }, + ]); + }); + + it("uses the configured default expiry", async () => { + const { client, calls } = fakeAssets(); + const publish = build(client, { defaultExpiresAfterMs: 1000 }); + + await publish(["a.txt", null], callContext()); + expect(calls[0]?.options).toEqual({ expiresAfter: 1000 }); + }); + + it.each([ + [[], /takes a path/], + [[""], /non-empty string/], + [["a.txt", "1h"], /options must be an object/], + [["a.txt", { expires: "1h" }], /unknown option "expires"/], + [["a.txt", { expiresAfter: -1 }], /positive number/], + [["a.txt", { expiresAfter: "soon" }], /not a duration/], + [["a.txt", { expiresAfter: true }], /number of milliseconds or a duration/], + [["a.txt", { disposition: "download" }], /inline" or "attachment/], + [["a.txt", { filename: 1 }], /filename must be a string/], + ])("rejects %j", async (args, message) => { + const { client, calls } = fakeAssets(); + const publish = build(client); + + await expect(publish(args as never, callContext())).rejects.toThrow(message); + expect(calls).toEqual([]); + }); + + it("does not publish once the call is cancelled", async () => { + const { client, calls } = fakeAssets(); + const publish = build(client); + const abort = new AbortController(); + abort.abort(); + + await expect(publish(["a.txt"], callContext({ signal: abort.signal }))).rejects.toThrow(); + expect(calls).toEqual([]); + }); + + it("fails to connect without an assets client", () => { + expect(() => build(undefined)).toThrow(/no assets client/); + }); + + it("rejects a non-positive default expiry", () => { + expect(() => createAssetsModule({ defaultExpiresAfterMs: 0 })).toThrow(/positive/); + }); + + it("describes itself", () => { + expect(createAssetsModule().description).toContain("publish(path"); + }); +}); diff --git a/packages/computer/src/modules/assets.ts b/packages/computer/src/modules/assets.ts new file mode 100644 index 00000000..bea48260 --- /dev/null +++ b/packages/computer/src/modules/assets.ts @@ -0,0 +1,110 @@ +import { parseDuration } from "../artifacts/duration.js"; +import type { ShareOptions } from "../assets/index.js"; +import type { + WorkspaceModuleFactory, + WorkspaceModuleFunctions, + WorkspaceModuleHost, + WorkspaceRuntimeValue, +} from "../runtime/types.js"; + +export interface AssetsModuleOptions { + readonly defaultExpiresAfterMs?: number; +} + +const DEFAULT_EXPIRES_AFTER_MS = 60 * 60 * 1000; +const PUBLISH_OPTION_KEYS = new Set(["expiresAfter", "filename", "disposition", "contentType"]); + +export function createAssetsModule(options: AssetsModuleOptions = {}): WorkspaceModuleFactory { + const defaultExpiresAfter = options.defaultExpiresAfterMs ?? DEFAULT_EXPIRES_AFTER_MS; + if (!Number.isFinite(defaultExpiresAfter) || defaultExpiresAfter <= 0) { + throw new Error("createAssetsModule: defaultExpiresAfterMs must be a positive number."); + } + + const create = (host: WorkspaceModuleHost): WorkspaceModuleFunctions => { + const assets = host.assets; + if (assets === undefined) { + throw new Error( + "ws:assets: the Workspace has no assets client. Pass `assets` to the Workspace.", + ); + } + return { + async publish(args, context) { + if (args.length === 0 || args.length > 2) { + throw new TypeError( + "publish(path, options?) takes a path and an optional options object.", + ); + } + const [path, publishOptions] = args; + if (typeof path !== "string" || path.length === 0) { + throw new TypeError("publish: path must be a non-empty string."); + } + const share = parsePublishOptions(publishOptions, defaultExpiresAfter); + const resolved = await context.resolvePath(path); + context.signal.throwIfAborted(); + return assets.share(resolved, share); + }, + }; + }; + return Object.assign(create, { + description: + '`publish(path, { expiresAfter?, filename?, disposition?, contentType? })` uploads a workspace file and returns a time-limited public URL. `expiresAfter` is milliseconds or a duration like "30m" or "1d"; the default is one hour and the maximum is seven days. `disposition` is "inline" or "attachment".', + }); +} + +function parsePublishOptions( + value: WorkspaceRuntimeValue | undefined, + defaultExpiresAfter: number, +): ShareOptions { + if (value === undefined || value === null) return { expiresAfter: defaultExpiresAfter }; + if (typeof value !== "object" || Array.isArray(value)) { + throw new TypeError("publish: options must be an object."); + } + for (const key of Object.keys(value)) { + if (!PUBLISH_OPTION_KEYS.has(key)) { + throw new TypeError( + `publish: unknown option ${JSON.stringify(key)}. Use expiresAfter, filename, disposition, or contentType.`, + ); + } + } + const share: ShareOptions = { + expiresAfter: parseExpiresAfter(value.expiresAfter, defaultExpiresAfter), + }; + const filename = optionalString(value.filename, "filename"); + if (filename !== undefined) share.filename = filename; + const contentType = optionalString(value.contentType, "contentType"); + if (contentType !== undefined) share.contentType = contentType; + const disposition = value.disposition; + if (disposition !== undefined && disposition !== null) { + if (disposition !== "inline" && disposition !== "attachment") { + throw new TypeError('publish: disposition must be "inline" or "attachment".'); + } + share.disposition = disposition; + } + return share; +} + +function parseExpiresAfter(value: WorkspaceRuntimeValue | undefined, fallback: number): number { + if (value === undefined || value === null) return fallback; + if (typeof value === "number") { + if (!Number.isFinite(value) || value <= 0) { + throw new TypeError("publish: expiresAfter must be a positive number of milliseconds."); + } + return value; + } + if (typeof value === "string") { + try { + return parseDuration(value) * 1000; + } catch (error) { + throw new TypeError( + `publish: expiresAfter ${JSON.stringify(value)} is not a duration like "30m": ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + throw new TypeError("publish: expiresAfter must be a number of milliseconds or a duration."); +} + +function optionalString(value: WorkspaceRuntimeValue | undefined, name: string) { + if (value === undefined || value === null) return undefined; + if (typeof value !== "string") throw new TypeError(`publish: ${name} must be a string.`); + return value; +} diff --git a/packages/computer/src/modules/container.test.ts b/packages/computer/src/modules/container.test.ts index 2001f67f..8acc1519 100644 --- a/packages/computer/src/modules/container.test.ts +++ b/packages/computer/src/modules/container.test.ts @@ -15,7 +15,7 @@ interface ExecOptions { readonly env?: Record; readonly stdin?: string; readonly timeoutMs: number; - readonly output?: { readonly maxBytes: number }; + readonly output?: { readonly maxBytes: number; readonly maxLines?: number }; } interface Run { @@ -137,6 +137,36 @@ describe("createContainerModule", () => { expect(runs[0]?.options.backend).toBe("linux"); }); + it("runs the prelude on its own line before each command", async () => { + const { runtime, runs } = fakeRuntime({}); + const container = build(runtime, { prelude: "set -o pipefail\nexport CI=1" }); + + await container.exec(["# comment\nnpm test | tee log"], callContext()); + expect(runs[0]?.command).toBe("set -o pipefail\nexport CI=1\n# comment\nnpm test | tee log"); + }); + + it("leaves the command alone with a blank prelude", async () => { + const { runtime, runs } = fakeRuntime({}); + const container = build(runtime, { prelude: " " }); + + await container.exec(["ls"], callContext()); + expect(runs[0]?.command).toBe("ls"); + }); + + it("checks the command before adding the prelude", async () => { + const { runtime, runs } = fakeRuntime({}); + const container = build(runtime, { prelude: "set -e" }); + + await expect(container.exec([" "], callContext())).rejects.toThrow(/non-empty string/); + expect(runs).toHaveLength(0); + }); + + it("rejects a prelude that is not a string", () => { + expect(() => createContainerModule({ prelude: 1 as never })).toThrow( + /prelude must be a string/, + ); + }); + it("refuses to run on a read-only backend", async () => { const { runtime, runs } = fakeRuntime({}); const container = build(runtime); @@ -219,6 +249,18 @@ describe("createContainerModule", () => { }); }); + it("asks the runtime to keep at most the configured lines", async () => { + const { runtime, runs } = fakeRuntime({}); + const container = build(runtime, { maxOutputBytes: 2048, maxOutputLines: 200 }); + + await container.exec(["npm test"], callContext()); + expect(runs[0]?.options.output).toEqual({ maxBytes: 2048, maxLines: 200 }); + }); + + it.each([0, -1, 1.5])("rejects maxOutputLines %s", (maxOutputLines) => { + expect(() => createContainerModule({ maxOutputLines })).toThrow(/maxOutputLines/); + }); + it.each([ ["no arguments", [], /takes a command/], ["too many arguments", ["ls", {}, {}], /takes a command/], diff --git a/packages/computer/src/modules/container.ts b/packages/computer/src/modules/container.ts index 63040fd6..5b69bd8c 100644 --- a/packages/computer/src/modules/container.ts +++ b/packages/computer/src/modules/container.ts @@ -42,6 +42,8 @@ export interface ContainerModuleOptions { * under the backend's `maxCapabilityBytes`. */ readonly maxOutputBytes?: number; + readonly maxOutputLines?: number; + readonly prelude?: string; } /** @@ -75,6 +77,20 @@ export function createContainerModule( if (!Number.isInteger(maxOutputBytes) || maxOutputBytes <= 0) { throw new Error("createContainerModule: maxOutputBytes must be a positive integer."); } + const maxOutputLines = options.maxOutputLines; + if (maxOutputLines !== undefined && (!Number.isInteger(maxOutputLines) || maxOutputLines <= 0)) { + throw new Error("createContainerModule: maxOutputLines must be a positive integer."); + } + const output = + maxOutputLines === undefined + ? { maxBytes: maxOutputBytes } + : { maxBytes: maxOutputBytes, maxLines: maxOutputLines }; + const prelude = options.prelude; + if (prelude !== undefined && typeof prelude !== "string") { + throw new Error("createContainerModule: prelude must be a string."); + } + const withPrelude = (command: string) => + prelude === undefined || prelude.trim() === "" ? command : `${prelude}\n${command}`; const create = (host: WorkspaceModuleHost): WorkspaceModuleFunctions => { // The factory runs when the JavaScript backend connects, so a @@ -105,14 +121,14 @@ export function createContainerModule( const timeoutMs = remainingTime(request.timeoutMs, context); context.signal.throwIfAborted(); - const handle = await host.runtime.exec(request.command, { + const handle = await host.runtime.exec(withPrelude(request.command), { backend, encoding: "utf8", timeoutMs, // The runtime keeps only the end of long output in memory and // saves the rest to a file, so a noisy command cannot exhaust // the Durable Object. - output: { maxBytes: maxOutputBytes }, + output, ...(request.cwd === undefined ? {} : { cwd: request.cwd }), ...(request.env === undefined ? {} : { env: request.env }), ...(request.stdin === undefined ? {} : { stdin: request.stdin }), diff --git a/packages/computer/src/modules/tools.test.ts b/packages/computer/src/modules/tools.test.ts new file mode 100644 index 00000000..3eb0f402 --- /dev/null +++ b/packages/computer/src/modules/tools.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; + +import type { WorkspaceModuleCallContext, WorkspaceModuleHost } from "../runtime/types.js"; +import { createToolsModule, type ModuleTool } from "./tools.js"; + +function callContext(signal = new AbortController().signal): WorkspaceModuleCallContext { + return { + signal, + deadline: Date.now() + 60_000, + access: "read-write", + resolvePath: async (path) => path, + }; +} + +function tool(name: string, calls: unknown[] = []): ModuleTool { + return { + name, + async execute(args) { + calls.push(args); + return { tool: name, args }; + }, + }; +} + +// SAFETY: The module uses nothing from the host. +const host = {} as WorkspaceModuleHost; + +describe("createToolsModule", () => { + it("exports each tool under its own name", async () => { + const calls: unknown[] = []; + const functions = createToolsModule(() => [tool("read", calls), tool("grep")])(host); + + expect(Object.keys(functions)).toEqual(["read", "grep"]); + await expect(functions.read?.([{ path: "a" }], callContext())).resolves.toEqual({ + tool: "read", + args: { path: "a" }, + }); + await functions.read?.([], callContext()); + expect(calls).toEqual([{ path: "a" }, {}]); + }); + + it("leaves out excluded tools and names that cannot be exports", () => { + const functions = createToolsModule( + () => [tool("exec"), tool("web-fetch"), tool("then"), tool("ls")], + { exclude: ["exec"] }, + )(host); + + expect(Object.keys(functions)).toEqual(["ls"]); + }); + + it("lists the tools when it is described, not when it is built", () => { + let names = ["read"]; + const module = createToolsModule(() => names.map((name) => tool(name)), { exclude: ["exec"] }); + + expect(module.description).toContain("Exports `read`."); + names = ["read", "grep", "exec"]; + expect(module.description).toContain("Exports `read`, `grep`."); + expect(module.description).toContain("There is no `exec` export."); + }); + + it.each([ + [[1], /must be an object/], + [[[1]], /must be an object/], + [[{}, {}], /takes one object/], + ])("rejects %j", async (args, message) => { + const calls: unknown[] = []; + const functions = createToolsModule(() => [tool("read", calls)])(host); + + await expect(functions.read?.(args as never, callContext())).rejects.toThrow(message); + expect(calls).toEqual([]); + }); + + it("does not run a tool once the call is cancelled", async () => { + const calls: unknown[] = []; + const functions = createToolsModule(() => [tool("read", calls)])(host); + const abort = new AbortController(); + abort.abort(new Error("cancelled")); + + await expect(functions.read?.([{}], callContext(abort.signal))).rejects.toThrow("cancelled"); + expect(calls).toEqual([]); + }); +}); diff --git a/packages/computer/src/modules/tools.ts b/packages/computer/src/modules/tools.ts new file mode 100644 index 00000000..dda1e27e --- /dev/null +++ b/packages/computer/src/modules/tools.ts @@ -0,0 +1,68 @@ +import type { + WorkspaceModuleFactory, + WorkspaceModuleFunction, + WorkspaceModuleFunctions, + WorkspaceRuntimeValue, +} from "../runtime/types.js"; + +export interface ModuleTool { + readonly name: string; + execute( + args: Record, + context: { signal: AbortSignal }, + ): Promise | WorkspaceRuntimeValue; +} + +export interface ToolsModuleOptions { + readonly exclude?: readonly string[]; +} + +const EXPORT_NAME = /^[A-Za-z_$][A-Za-z0-9_$]*$/; +const RESERVED = new Set(["default", "then"]); + +export function createToolsModule( + tools: () => readonly ModuleTool[], + options: ToolsModuleOptions = {}, +): WorkspaceModuleFactory { + const excluded = new Set(options.exclude ?? []); + const available = () => + tools().filter( + (tool) => !excluded.has(tool.name) && EXPORT_NAME.test(tool.name) && !RESERVED.has(tool.name), + ); + const create = (): WorkspaceModuleFunctions => { + const functions: Record = {}; + for (const tool of available()) { + functions[tool.name] = async (args, context) => { + if (args.length > 1) { + throw new TypeError(`${tool.name}(arguments) takes one object of the tool's arguments.`); + } + const [input] = args; + if ( + input !== undefined && + input !== null && + (typeof input !== "object" || Array.isArray(input)) + ) { + throw new TypeError(`${tool.name}: arguments must be an object.`); + } + context.signal.throwIfAborted(); + return tool.execute(input ?? {}, { signal: context.signal }); + }; + } + return functions; + }; + return Object.defineProperty(create, "description", { + enumerable: true, + get() { + const names = available().map((tool) => `\`${tool.name}\``); + const listed = + names.length === 0 ? "No tools are available." : `Exports ${names.join(", ")}.`; + const left = [...excluded].map((name) => `\`${name}\``); + return [ + "The agent's own tools, callable from code under the same names.", + listed, + "Each takes one object of the tool's arguments.", + ...(left.length === 0 ? [] : [`There is no ${left.join(" or ")} export.`]), + ].join(" "); + }, + }) as WorkspaceModuleFactory; +} diff --git a/packages/computer/src/runtime/types.ts b/packages/computer/src/runtime/types.ts index 82968cee..76538da0 100644 --- a/packages/computer/src/runtime/types.ts +++ b/packages/computer/src/runtime/types.ts @@ -50,6 +50,7 @@ export interface WorkspaceModuleHost { readonly artifacts: import("../artifacts/index.js").ArtifactClient; /** The Workspace runtime, for running commands on other backends. */ readonly runtime: import("./runtime.js").WorkspaceRuntime; + readonly assets?: import("../assets/index.js").AssetsClient; } /** @@ -296,6 +297,7 @@ export interface WorkspaceModuleBackendHandle { export type WorkspaceModuleBackendHost = import("../backend.js").WorkspaceBackendHost & { /** The Workspace runtime, handed to host modules. */ readonly runtime: import("./runtime.js").WorkspaceRuntime; + readonly assets?: import("../assets/index.js").AssetsClient; }; export interface WorkspaceModuleBackend { diff --git a/packages/computer/src/tools/common/exec.ts b/packages/computer/src/tools/common/exec.ts index ba6e4b21..c31a9ffd 100644 --- a/packages/computer/src/tools/common/exec.ts +++ b/packages/computer/src/tools/common/exec.ts @@ -140,7 +140,7 @@ export interface ExecInput { cwd?: string; backend?: string; env?: Record; - input?: WorkspaceRuntimeValue; + input?: { [key: string]: WorkspaceRuntimeValue }; } export interface ExecCallContext { @@ -151,6 +151,7 @@ export interface ExecCallContext { export interface ExecDefinition { description: string; inputSchema: z.ZodType; + prepareArguments(args: unknown): unknown; /** * Yields running snapshots while the command streams, then one * terminal snapshot. Every snapshot is a complete result, so a @@ -213,12 +214,13 @@ export function defineExec(options: ExecToolOptions): ExecDefinition { ); } if (callableBackendIds.size > 0) { - shape.input = jsonValueSchema + shape.input = z + .record(z.string(), jsonValueSchema) .optional() .describe( single - ? "Structured value handed to the module." - : "Structured value handed to a callable backend's module. Other backends reject it.", + ? "Input object handed to the module." + : "Input object handed to a callable backend's module. Other backends reject it.", ); } // SAFETY: Every field in `shape` has the type ExecInput gives it, and the fields left out are optional there. @@ -227,6 +229,7 @@ export function defineExec(options: ExecToolOptions): ExecDefinition { return { description, inputSchema, + prepareArguments: callableBackendIds.size > 0 ? parseInputObject : (args) => args, execute: async function* ({ command, cwd, backend, env, input }, { abortSignal } = {}) { // With one backend there is nothing to choose. With several the // schema requires `backend`; a caller that skips the schema gets @@ -351,7 +354,7 @@ function outputHint(limits: OutputLimits): string { } const SHELL_HINT = "Use for builds, test runs, typechecks, formatters, and git plumbing."; const CALLABLE_HINT = - "Pass `input` to hand the module a structured value, and read its return value back from the `result` field."; + "Pass an `input` object to hand the module structured values, and read its return value back from the `result` field."; interface DescribedBackend { readonly id: string; @@ -438,6 +441,20 @@ function commandHint(backends: readonly DescribedBackend[]): string { return "Shell command, e.g. 'npm test' or 'git diff HEAD'. For a callable backend this is the module source to run."; } +function parseInputObject(args: unknown): unknown { + if (args === null || typeof args !== "object" || Array.isArray(args)) return args; + const input = (args as { input?: unknown }).input; + if (typeof input !== "string" || !input.trimStart().startsWith("{")) return args; + let parsed: unknown; + try { + parsed = JSON.parse(input); + } catch { + return args; + } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return args; + return { ...args, input: parsed }; +} + function errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } diff --git a/packages/computer/src/tools/common/fs/confine.test.ts b/packages/computer/src/tools/common/fs/confine.test.ts new file mode 100644 index 00000000..edb9afca --- /dev/null +++ b/packages/computer/src/tools/common/fs/confine.test.ts @@ -0,0 +1,95 @@ +import { SQLiteTestStorage } from "@cloudflare/dofs/testing"; +import { describe, expect, it } from "vitest"; + +import { Workspace } from "../../../workspace.js"; +import { createPiTools } from "../../pi-ai/index.js"; +import { normalizeRootedPath } from "./confine.js"; + +async function setup() { + const workspace = new Workspace({ storage: new SQLiteTestStorage(), now: () => 0 }); + await workspace.fs.mkdir("/workspace/src", { recursive: true }); + await workspace.fs.mkdir("/secret", { recursive: true }); + await workspace.fs.writeFile("/workspace/src/a.txt", new TextEncoder().encode("inside\n")); + await workspace.fs.writeFile("/secret/key.txt", new TextEncoder().encode("outside\n")); + const tools = createPiTools({ workspace, root: "/workspace" }); + const call = (name: string, args: Record) => + tools.execute({ id: crypto.randomUUID(), name, arguments: args }); + const text = (result: { content: Array<{ type: string; text?: string }> }) => + result.content.map((part) => part.text ?? "").join(""); + return { workspace, call, text }; +} + +describe("normalizeRootedPath", () => { + it.each([ + ["/workspace", "src/a.txt", "/workspace/src/a.txt"], + ["/workspace", "./src/../src/a.txt", "/workspace/src/a.txt"], + ["/workspace/", "/workspace", "/workspace"], + ["/", "/anything", "/anything"], + ])("resolves %s + %s to %s", (root, input, expected) => { + expect(normalizeRootedPath(root, input)).toBe(expected); + }); + + it.each([ + ["/workspace", "/secret/key.txt"], + ["/workspace", "../secret/key.txt"], + ["/workspace", "/workspace-other/x"], + ])("rejects %s + %s", (root, input) => { + expect(() => normalizeRootedPath(root, input)).toThrow(/outside \/workspace/); + }); +}); + +describe("tools with a root", () => { + it("reads a path relative to the root", async () => { + const { call, text } = await setup(); + + const result = await call("read", { path: "src/a.txt" }); + + expect(result.isError).toBe(false); + expect(text(result)).toBe("inside"); + }); + + it("refuses paths outside the root in every file tool", async () => { + const { workspace, call, text } = await setup(); + + for (const [name, args] of [ + ["read", { path: "/secret/key.txt" }], + ["read", { path: "../secret/key.txt" }], + ["ls", { path: "/secret" }], + ["find", { path: "/secret", pattern: "**" }], + ["grep", { path: "/secret", query: "outside" }], + ["write", { path: "/secret/new.txt", content: "x" }], + ["edit", { path: "/secret/key.txt", edits: [{ oldText: "outside", newText: "x" }] }], + ["delete", { path: "/secret/key.txt" }], + ] as const) { + const result = await call(name, args); + expect(result.isError, name).toBe(true); + expect(text(result), name).toContain("outside /workspace"); + } + const key = await new Response(await workspace.fs.readFile("/secret/key.txt")).text(); + expect(key).toBe("outside\n"); + await expect(workspace.fs.stat("/secret/new.txt")).rejects.toThrow(); + }); + + it("refuses a path through a symbolic link", async () => { + const { workspace, call, text } = await setup(); + await workspace.fs.symlink("/secret", "/workspace/link"); + + const read = await call("read", { path: "/workspace/link/key.txt" }); + const write = await call("write", { path: "link/new.txt", content: "x" }); + + expect(read.isError).toBe(true); + expect(text(read)).toContain("symbolic link: /workspace/link"); + expect(write.isError).toBe(true); + await expect(workspace.fs.stat("/secret/new.txt")).rejects.toThrow(); + }); + + it("writes a new file under a directory that does not exist yet", async () => { + const { workspace, call } = await setup(); + + const result = await call("write", { path: "new/dir/b.txt", content: "made" }); + + expect(result.isError).toBe(false); + const made = await new Response(await workspace.fs.readFile("/workspace/new/dir/b.txt")).text(); + expect(made).toBe("made"); + }); +}); diff --git a/packages/computer/src/tools/common/fs/confine.ts b/packages/computer/src/tools/common/fs/confine.ts new file mode 100644 index 00000000..8aba4ba8 --- /dev/null +++ b/packages/computer/src/tools/common/fs/confine.ts @@ -0,0 +1,113 @@ +const PATH_ARGUMENT: Readonly> = { + stat: 0, + lstat: 0, + readFile: 0, + writeFile: 0, + mkdir: 0, + rm: 0, + find: 0, + readdir: 0, + readlink: 0, + grep: 1, +}; + +interface ConfinableFs { + lstat?(path: string): Promise<{ isSymbolicLink?: boolean }>; +} + +interface ConfinableWorkspace { + fs: object; + runtime?: unknown; + assets?: { share(path: string, options: never): Promise }; + sessionId?: string | null; +} + +export function normalizeRootedPath(root: string, input: string): string { + const base = normalizeAbsolute(root); + const absolute = normalizeAbsolute(input.startsWith("/") ? input : `${base}/${input}`); + if (base !== "/" && absolute !== base && !absolute.startsWith(`${base}/`)) { + throw new Error(`Path is outside ${base}: ${input}`); + } + return absolute; +} + +function normalizeAbsolute(path: string): string { + const parts: string[] = []; + for (const part of path.split("/")) { + if (part === "" || part === ".") continue; + if (part === "..") parts.pop(); + else parts.push(part); + } + return `/${parts.join("/")}`; +} + +const confinedFilesystems = new WeakMap>(); + +export function confineWorkspace(workspace: W, root: string): W { + const base = normalizeAbsolute(root); + const fs = workspace.fs as ConfinableFs & Record; + const byRoot = confinedFilesystems.get(fs) ?? new Map(); + confinedFilesystems.set(fs, byRoot); + + const resolve = async (input: unknown): Promise => { + if (typeof input !== "string") throw new TypeError("path must be a string"); + const path = normalizeRootedPath(base, input); + if (typeof fs.lstat !== "function") return path; + let current = base === "/" ? "" : base; + for (const part of path.slice(current.length).split("/").filter(Boolean)) { + current = `${current}/${part}`; + let info: { isSymbolicLink?: boolean }; + try { + info = await fs.lstat(current); + } catch (error) { + if (isMissing(error)) return path; + throw error; + } + if (info.isSymbolicLink === true) { + throw new Error(`Path contains a symbolic link: ${current}`); + } + } + return path; + }; + + const confinedFs = + byRoot.get(base) ?? + new Proxy(fs, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + const index = typeof property === "string" ? PATH_ARGUMENT[property] : undefined; + if (index === undefined) return value.bind(target); + return async (...args: unknown[]) => { + const next = [...args]; + next[index] = await resolve(args[index]); + return value.apply(target, next); + }; + }, + }); + byRoot.set(base, confinedFs); + + const assets = workspace.assets; + const confinedAssets = + assets === undefined + ? undefined + : { + share: async (path: string, options: never) => assets.share(await resolve(path), options), + }; + + return new Proxy(workspace, { + get(target, property) { + if (property === "fs") return confinedFs; + if (property === "assets") return confinedAssets; + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); +} + +function isMissing(error: unknown): boolean { + if (typeof error !== "object" || error === null) return false; + const candidate = error as { code?: unknown; message?: unknown }; + if (candidate.code === "ENOENT") return true; + return typeof candidate.message === "string" && /ENOENT|no such/i.test(candidate.message); +} diff --git a/packages/computer/src/tools/common/fs/edit.test.ts b/packages/computer/src/tools/common/fs/edit.test.ts new file mode 100644 index 00000000..0e3368ba --- /dev/null +++ b/packages/computer/src/tools/common/fs/edit.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest"; + +import { editInStore } from "./edit.js"; +import type { FileStore } from "./types.js"; + +function memoryStore(initial: string) { + let content = new TextEncoder().encode(initial); + const store: FileStore = { + async stat() { + return { size: content.byteLength, mtime: 1, mode: 0o100644 }; + }, + async *readChunks() { + yield content; + }, + async readAll() { + return content; + }, + async write(_path, next) { + content = next; + }, + }; + return { store, text: () => new TextDecoder().decode(content) }; +} + +const lines = (count: number, prefix: string) => + Array.from({ length: count }, (_, index) => `${prefix}${index}`).join("\n"); + +describe("editInStore diff bounds", () => { + it("returns the diff and patch of a small edit", async () => { + const { store, text } = memoryStore("a\nb\nc\n"); + + const result = await editInStore( + { store }, + { path: "/w/f.txt", edits: [{ oldText: "b", newText: "B" }] }, + ); + + expect(text()).toBe("a\nB\nc\n"); + expect(result).toMatchObject({ path: "/w/f.txt", editsApplied: 1, firstChangedLine: 2 }); + expect(result).not.toHaveProperty("diffTruncated"); + expect((result as { patch: string }).patch).toContain("-b\n+B"); + }); + + it("applies an edit too large to diff and says the diff was skipped", async () => { + const before = lines(50, "old"); + const { store, text } = memoryStore(`head\n${before}\ntail\n`); + + const result = await editInStore( + { store, maxDiffLines: 10 }, + { path: "/w/f.txt", edits: [{ oldText: before, newText: lines(50, "new") }] }, + ); + + expect(text()).toContain("new49"); + expect(result).toEqual({ + path: "/w/f.txt", + editsApplied: 1, + diff: "", + patch: "", + firstChangedLine: undefined, + diffTruncated: true, + }); + }); + + it("counts replaced lines on either side of every edit", async () => { + const { store } = memoryStore("x\ny\n"); + + const result = await editInStore( + { store, maxDiffLines: 5 }, + { + path: "/w/f.txt", + edits: [ + { oldText: "x", newText: lines(3, "a") }, + { oldText: "y", newText: lines(3, "b") }, + ], + }, + ); + + expect(result).toMatchObject({ diffTruncated: true, diff: "" }); + }); + + it("cuts a long diff and patch on a character boundary", async () => { + const { store } = memoryStore("é".repeat(400)); + + const result = (await editInStore( + { store, maxDiffBytes: 101 }, + { path: "/w/f.txt", edits: [{ oldText: "é".repeat(400), newText: "ü".repeat(400) }] }, + )) as { diff: string; patch: string; diffTruncated?: boolean }; + + expect(result.diffTruncated).toBe(true); + expect(new TextEncoder().encode(result.diff).byteLength).toBeLessThanOrEqual(101); + expect(new TextEncoder().encode(result.patch).byteLength).toBeLessThanOrEqual(101); + expect(result.diff).not.toContain("\uFFFD"); + expect(result.patch).not.toContain("\uFFFD"); + }); +}); diff --git a/packages/computer/src/tools/common/fs/edit.ts b/packages/computer/src/tools/common/fs/edit.ts index ffd00f6f..b4a6571a 100644 --- a/packages/computer/src/tools/common/fs/edit.ts +++ b/packages/computer/src/tools/common/fs/edit.ts @@ -20,9 +20,13 @@ export interface EditToolOptions { * Default 2 MiB. */ maxBytes?: number; + maxDiffLines?: number; + maxDiffBytes?: number; } const DEFAULT_MAX_BYTES = 2 * 1024 * 1024; +const DEFAULT_MAX_DIFF_LINES = 2_000; +const DEFAULT_MAX_DIFF_BYTES = 128 * 1024; const replacementSchema = z .object({ @@ -59,6 +63,7 @@ export const editOutputSchema = z.union([ diff: z.string(), patch: z.string(), firstChangedLine: z.number().int().optional(), + diffTruncated: z.boolean().optional(), }), z.object({ error: z.string() }), ]); @@ -78,6 +83,7 @@ export interface EditSuccess { patch: string; /** Undefined when the edit produced no line-level change. */ firstChangedLine: number | undefined; + diffTruncated?: boolean; } export type EditResult = EditSuccess | { error: string }; @@ -122,6 +128,8 @@ export async function editInStore( ): Promise { const { store } = options; const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; + const maxDiffLines = options.maxDiffLines ?? DEFAULT_MAX_DIFF_LINES; + const maxDiffBytes = options.maxDiffBytes ?? DEFAULT_MAX_DIFF_BYTES; const { path, edits } = prepareArguments(rawInput); if (!Array.isArray(edits) || edits.length === 0) { @@ -161,18 +169,51 @@ export async function editInStore( // that case so the store applies its own default. await store.write(path, new TextEncoder().encode(finalContent), { mode: stat.mode }); + const replacedLines = edits.reduce( + (total, edit) => total + Math.max(countLines(edit.oldText), countLines(edit.newText)), + 0, + ); + if (replacedLines > maxDiffLines) { + return { + path, + editsApplied: edits.length, + diff: "", + patch: "", + firstChangedLine: undefined, + diffTruncated: true, + }; + } + const diffResult = generateDiffString(baseContent, newContent); - const patch = generateUnifiedPatch(path, baseContent, newContent); + const diff = truncateUtf8(diffResult.diff, maxDiffBytes); + const patch = truncateUtf8(generateUnifiedPatch(path, baseContent, newContent), maxDiffBytes); return { path, editsApplied: edits.length, - diff: diffResult.diff, - patch, + diff: diff.text, + patch: patch.text, firstChangedLine: diffResult.firstChangedLine, + ...(diff.truncated || patch.truncated ? { diffTruncated: true } : {}), }; } catch (err) { return { error: err instanceof Error ? err.message : String(err) }; } }); } + +function countLines(value: string): number { + let lines = 1; + for (let index = value.indexOf("\n"); index !== -1; index = value.indexOf("\n", index + 1)) { + lines += 1; + } + return lines; +} + +function truncateUtf8(value: string, maxBytes: number): { text: string; truncated: boolean } { + const encoded = new TextEncoder().encode(value); + if (encoded.byteLength <= maxBytes) return { text: value, truncated: false }; + let end = maxBytes; + while (end > 0 && (encoded[end] & 0xc0) === 0x80) end -= 1; + return { text: new TextDecoder().decode(encoded.subarray(0, end)), truncated: true }; +} diff --git a/packages/computer/src/tools/common/fs/find.ts b/packages/computer/src/tools/common/fs/find.ts index 6cae445a..4a9f3ee2 100644 --- a/packages/computer/src/tools/common/fs/find.ts +++ b/packages/computer/src/tools/common/fs/find.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { globList } from "./globs.js"; interface FoundEntry { path: string; @@ -28,7 +29,7 @@ export const findInputSchema = z.object({ .string() .describe('Glob pattern relative to path, for example "**/*.ts" or "src/?.js".'), exclude: z - .array(z.string()) + .union([z.string(), z.array(z.string())]) .optional() .describe( 'Glob patterns to leave out, for example ["node_modules/**", "**/.git/**"]. An excluded directory is skipped along with everything below it.', @@ -43,7 +44,7 @@ export const findDescription = export interface FindInput { path?: string; pattern: string; - exclude?: string[]; + exclude?: string | string[]; limit?: number; offset?: number; } @@ -76,7 +77,7 @@ export async function findInWorkspace( const matches = await workspace.fs.find(directory, pattern, { limit: pageSize + 1, offset: pageOffset, - exclude, + exclude: globList(exclude), }); const truncated = matches.length > pageSize; const entries = truncated ? matches.slice(0, pageSize) : matches; diff --git a/packages/computer/src/tools/common/fs/globs.ts b/packages/computer/src/tools/common/fs/globs.ts new file mode 100644 index 00000000..ffe11720 --- /dev/null +++ b/packages/computer/src/tools/common/fs/globs.ts @@ -0,0 +1,11 @@ +export function globList(value: string | readonly string[] | undefined): string[] | undefined { + if (value === undefined) return undefined; + const list = (typeof value === "string" ? [value] : value).filter((glob) => glob.trim() !== ""); + return list.length === 0 ? undefined : list; +} + +export function globOne(value: string | readonly string[] | undefined): string | undefined { + const list = globList(value); + if (list === undefined) return undefined; + return list.length === 1 ? list[0] : `{${list.join(",")}}`; +} diff --git a/packages/computer/src/tools/common/fs/grep.test.ts b/packages/computer/src/tools/common/fs/grep.test.ts new file mode 100644 index 00000000..23b2b554 --- /dev/null +++ b/packages/computer/src/tools/common/fs/grep.test.ts @@ -0,0 +1,88 @@ +import { SQLiteTestStorage } from "@cloudflare/dofs/testing"; +import { describe, expect, it } from "vitest"; + +import { Workspace } from "../../../workspace.js"; +import { findInWorkspace } from "./find.js"; +import { grepInputSchema, grepInWorkspace } from "./grep.js"; + +async function makeWorkspace(): Promise { + const workspace = new Workspace({ storage: new SQLiteTestStorage(), now: () => 0 }); + await workspace.fs.mkdir("/workspace/src", { recursive: true }); + await workspace.fs.mkdir("/workspace/node_modules/dep", { recursive: true }); + const write = (path: string, text: string) => + workspace.fs.writeFile(path, new TextEncoder().encode(text)); + await write("/workspace/src/a.ts", "needle\n"); + await write("/workspace/src/b.md", "needle\n"); + await write("/workspace/src/c.json", "needle\n"); + await write("/workspace/node_modules/dep/index.ts", "needle\n"); + return workspace; +} + +function paths(result: Awaited>): string[] { + if ("error" in result) throw new Error(result.error); + return result.matches.map((match) => match.path).sort(); +} + +describe("grepInWorkspace globs", () => { + it("takes several include globs", async () => { + const workspace = await makeWorkspace(); + + const result = await grepInWorkspace(workspace, { + query: "needle", + include: ["src/*.ts", "src/*.md"], + }); + + expect(paths(result)).toEqual(["/workspace/src/a.ts", "/workspace/src/b.md"]); + }); + + it("takes one exclude glob as a string", async () => { + const workspace = await makeWorkspace(); + + const result = await grepInWorkspace(workspace, { + query: "needle", + exclude: "node_modules/**", + }); + + expect(paths(result)).toEqual([ + "/workspace/src/a.ts", + "/workspace/src/b.md", + "/workspace/src/c.json", + ]); + }); + + it("ignores blank globs", async () => { + const workspace = await makeWorkspace(); + + const result = await grepInWorkspace(workspace, { + query: "needle", + include: [" "], + exclude: [""], + }); + + expect(paths(result)).toHaveLength(4); + }); + + it("accepts a string or a list in its schema", () => { + expect( + grepInputSchema.safeParse({ query: "x", include: "*.ts", exclude: "a/**" }).success, + ).toBe(true); + expect( + grepInputSchema.safeParse({ query: "x", include: ["*.ts"], exclude: ["a/**"] }).success, + ).toBe(true); + expect(grepInputSchema.safeParse({ query: "x", include: 1 }).success).toBe(false); + }); +}); + +describe("findInWorkspace globs", () => { + it("takes one exclude glob as a string", async () => { + const workspace = await makeWorkspace(); + + const result = await findInWorkspace(workspace, { + pattern: "**/*.ts", + exclude: "node_modules/**", + }); + + if ("error" in result) throw new Error(result.error); + expect(result.entries.map((entry) => entry.path)).toEqual(["/workspace/src/a.ts"]); + }); +}); diff --git a/packages/computer/src/tools/common/fs/grep.ts b/packages/computer/src/tools/common/fs/grep.ts index 6f3f6cd9..aa33fe7b 100644 --- a/packages/computer/src/tools/common/fs/grep.ts +++ b/packages/computer/src/tools/common/fs/grep.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { globList, globOne } from "./globs.js"; interface GrepContextLine { line: number; @@ -40,11 +41,13 @@ export const grepInputSchema = z.object({ path: z.string().default("/workspace").describe("Absolute file or directory to search."), query: z.string().describe("Literal string or regular expression to search for."), include: z - .string() + .union([z.string(), z.array(z.string())]) .optional() - .describe('Glob relative to path that limits searched files, for example "**/*.ts".'), + .describe( + 'Glob relative to path that limits searched files, for example "**/*.ts" or ["**/*.ts", "**/*.md"].', + ), exclude: z - .array(z.string()) + .union([z.string(), z.array(z.string())]) .optional() .describe( 'Glob patterns to leave out, for example ["node_modules/**", "**/.git/**"]. An excluded directory is skipped along with everything below it.', @@ -62,8 +65,8 @@ export const grepDescription = export interface GrepInput { path?: string; query: string; - include?: string; - exclude?: string[]; + include?: string | string[]; + exclude?: string | string[]; regex?: boolean; ignoreCase?: boolean; context?: number; @@ -103,8 +106,8 @@ export async function grepInWorkspace( }; const matches = await workspace.fs.grep(query, target, { ...searchOptions, - include, - exclude, + include: globOne(include), + exclude: globList(exclude), limit: pageSize + 1, offset: pageOffset, }); diff --git a/packages/computer/src/tools/common/options.ts b/packages/computer/src/tools/common/options.ts index 27b4e510..5d20d1e7 100644 --- a/packages/computer/src/tools/common/options.ts +++ b/packages/computer/src/tools/common/options.ts @@ -1,4 +1,5 @@ import type { ExecBackends, ExecToolOptions, ExecWorkspaceLike } from "./exec.js"; +import { confineWorkspace } from "./fs/confine.js"; import type { EditToolOptions } from "./fs/edit.js"; import type { ReadToolOptions } from "./fs/read.js"; import { type WorkspaceLike as FileWorkspaceLike, WorkspaceFileStore } from "./fs/store.js"; @@ -10,6 +11,7 @@ export interface CreateToolsOptions { workspace: FileWorkspaceLike & Partial & Partial; /** Omit `write`, `edit`, `delete`, `exec`, and `publish`. */ readonly?: boolean; + root?: string; /** Set `false` to omit `publish` even when assets are configured. */ assets?: boolean; read?: Omit; @@ -21,6 +23,7 @@ export interface CreateToolsOptions { * Workspace has; `{}` means no exec tool. */ exec?: ExecBackends; + execOutput?: Pick; /** * @deprecated Use `exec`. `{ backends }` becomes `exec: backends`; * `defaultBackend` is ignored, because the model names a backend @@ -48,7 +51,11 @@ export interface ResolvedToolOptions { /** Resolve the options into what each tool needs, so every tool set offers the same tools. */ export function resolveToolOptions(options: CreateToolsOptions): ResolvedToolOptions { - const store = new WorkspaceFileStore(options.workspace); + const workspace = + options.root === undefined + ? options.workspace + : confineWorkspace(options.workspace, options.root); + const store = new WorkspaceFileStore(workspace); const readonly = options.readonly === true; return { read: { store, ...options.read }, @@ -56,9 +63,9 @@ export function resolveToolOptions(options: CreateToolsOptions): ResolvedToolOpt edit: { store, ...options.edit }, delete: { store }, exec: readonly ? undefined : execOptions(options), - publish: !readonly && options.assets !== false && options.workspace.assets !== undefined, + publish: !readonly && options.assets !== false && workspace.assets !== undefined, readonly, - workspace: options.workspace, + workspace, }; } @@ -68,7 +75,13 @@ function execOptions(options: CreateToolsOptions): ExecToolOptions | undefined { if (runtime === undefined) return undefined; const exec = selectExec(options, runtime); if (Object.keys(exec.backends).length === 0) return undefined; - return { workspace: { runtime }, ...exec }; + const output = options.execOutput; + return { + workspace: { runtime }, + ...exec, + ...(output?.maxBytes === undefined ? {} : { maxBytes: output.maxBytes }), + ...(output?.maxLines === undefined ? {} : { maxLines: output.maxLines }), + }; } function selectExec( diff --git a/packages/computer/src/tools/core.test.ts b/packages/computer/src/tools/core.test.ts new file mode 100644 index 00000000..dcedeb4a --- /dev/null +++ b/packages/computer/src/tools/core.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; + +import { defineExec, type ExecToolOutput } from "./core.js"; + +describe("defineExec", () => { + it("parses an input object sent as text only for a callable backend", () => { + const define = (callable: boolean) => + defineExec({ + workspace: { + runtime: { + backends: () => [{ id: "b", protocol: "module", callable }], + exec: async () => { + throw new Error("not run"); + }, + }, + }, + }); + const args = { command: "x", input: '{"a":1}' }; + + expect(define(true).prepareArguments(args)).toEqual({ command: "x", input: { a: 1 } }); + expect(args.input).toBe('{"a":1}'); + expect(define(false).prepareArguments(args)).toBe(args); + expect(define(true).prepareArguments({ command: "x", input: "null" })).toEqual({ + command: "x", + input: "null", + }); + expect(define(true).prepareArguments("text")).toBe("text"); + }); + + it("builds the exec tool with no agent library", async () => { + const exec = defineExec({ + workspace: { + runtime: { + backends: () => [{ id: "sh", protocol: "command", callable: false }], + exec: async () => ({ + result: async () => ({ exitCode: 0, stdout: "hi\n", stderr: "" }), + }), + }, + }, + }); + + expect(exec.description).toContain("Run a shell command"); + expect(exec.inputSchema.safeParse({ command: "echo hi" }).success).toBe(true); + const snapshots: ExecToolOutput[] = []; + for await (const snapshot of exec.execute({ command: "echo hi" })) snapshots.push(snapshot); + expect(snapshots.at(-1)).toEqual({ + command: "echo hi", + cwd: null, + backend: "sh", + exitCode: 0, + stdout: "hi\n", + stderr: "", + }); + }); +}); diff --git a/packages/computer/src/tools/core.ts b/packages/computer/src/tools/core.ts new file mode 100644 index 00000000..d086afbf --- /dev/null +++ b/packages/computer/src/tools/core.ts @@ -0,0 +1,13 @@ +export { + defineExec, + type ExecBackendOptions, + type ExecBackends, + type ExecCallContext, + type ExecDefinition, + type ExecInput, + type ExecRuntimeHandle, + type ExecStreamEvent, + type ExecToolOptions, + type ExecToolOutput, + type ExecWorkspaceLike, +} from "./common/exec.js"; diff --git a/packages/computer/src/tools/pi-ai/index.test.ts b/packages/computer/src/tools/pi-ai/index.test.ts index 83c0e063..5d495233 100644 --- a/packages/computer/src/tools/pi-ai/index.test.ts +++ b/packages/computer/src/tools/pi-ai/index.test.ts @@ -211,6 +211,43 @@ describe("createPiTools execution", () => { expect(parsed.entries[0].name).toBe("a.txt"); }); + it("returns the tool's own output as details", async () => { + const workspace = makeWorkspace(); + (workspace.runtime as unknown as Record).exec = async () => ({ + result: async () => ({ exitCode: 2, stdout: "out", stderr: "err" }), + }); + fakeBackends(workspace, [{ id: "sh", callable: false }]); + const tools = createPiTools({ workspace, exec: { sh: {} } }); + + await tools.execute({ id: "1", name: "write", arguments: { path: "/w/a.txt", content: "x" } }); + const ls = await tools.execute({ id: "2", name: "ls", arguments: { path: "/w" } }); + const exec = await tools.execute({ id: "3", name: "exec", arguments: { command: "make" } }); + const missing = await tools.execute({ + id: "4", + name: "read", + arguments: { path: "/w/missing.txt" }, + }); + + expect(ls.details).toEqual(JSON.parse((ls.content[0] as { text: string }).text)); + expect(exec.details).toEqual({ + command: "make", + cwd: null, + backend: "sh", + exitCode: 2, + stdout: "out", + stderr: "err", + }); + expect(missing).toMatchObject({ isError: true, details: { error: expect.any(String) } }); + }); + + it("leaves details off a result that fails validation", async () => { + const tools = createPiTools({ workspace: makeWorkspace() }); + + const result = await tools.execute({ id: "1", name: "read", arguments: { path: 42 } }); + + expect(result).not.toHaveProperty("details"); + }); + it("marks a missing file as an error result", async () => { const tools = createPiTools({ workspace: makeWorkspace() }); @@ -242,8 +279,7 @@ describe("createPiTools execution", () => { expect((result.content[0] as { text: string }).text).toContain('Unknown tool "nope"'); }); - it("keeps a null the tool genuinely accepts", async () => { - // `exec`'s structured input is any JSON value, so null means null. + it("hands exec an input object, and treats a null input as absent", async () => { const seen: Array<{ input: unknown }> = []; const workspace = makeWorkspace(); (workspace.runtime as unknown as Record).exec = async ( @@ -256,11 +292,88 @@ describe("createPiTools execution", () => { fakeBackends(workspace, [{ id: "js", callable: true, description: "callable" }]); const tools = createPiTools({ workspace }); - await tools.execute({ id: "1", name: "exec", arguments: { command: "a", input: null } }); - await tools.execute({ id: "2", name: "exec", arguments: { command: "b" } }); + const exec = declaration(tools, "exec"); + const input = exec.parameters.properties?.input as { type?: string } | undefined; + expect(input?.type).toBe("object"); + await tools.execute({ id: "1", name: "exec", arguments: { command: "a", input: { n: [1] } } }); + await tools.execute({ id: "2", name: "exec", arguments: { command: "b", input: null } }); + const array = await tools.execute({ + id: "3", + name: "exec", + arguments: { command: "c", input: [1, 2] }, + }); + + expect(seen).toEqual([{ input: { n: [1] } }, { input: undefined }]); + expect(array.isError).toBe(true); + }); + + it("parses an input object sent as JSON text", async () => { + const seen: unknown[] = []; + const workspace = makeWorkspace(); + (workspace.runtime as unknown as Record).exec = async ( + _command: string, + options: { input?: unknown }, + ) => { + seen.push(options.input); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }; + fakeBackends(workspace, [{ id: "js", callable: true, description: "callable" }]); + const tools = createPiTools({ workspace }); + + const text = await tools.execute({ + id: "1", + name: "exec", + arguments: { command: "a", input: ' {"dir": "/workspace"}' }, + }); + const array = await tools.execute({ + id: "2", + name: "exec", + arguments: { command: "b", input: "[1]" }, + }); + const broken = await tools.execute({ + id: "3", + name: "exec", + arguments: { command: "c", input: "{dir:" }, + }); + + expect(text.isError).toBe(false); + expect(seen).toEqual([{ dir: "/workspace" }]); + expect(array.isError).toBe(true); + expect(broken.isError).toBe(true); + }); + + it("cuts exec output by the limits `execOutput` sets", async () => { + const seen: unknown[] = []; + const workspace = makeWorkspace(); + (workspace.runtime as unknown as Record).exec = async ( + _command: string, + options: { output?: unknown }, + ) => { + seen.push(options.output); + return { result: async () => ({ exitCode: 0, stdout: "", stderr: "" }) }; + }; + fakeBackends(workspace, [{ id: "sh", callable: false }]); + const tools = createPiTools({ + workspace, + exec: { sh: {} }, + execOutput: { maxLines: 200, maxBytes: 4096 }, + }); + + expect(declaration(tools, "exec").description).toContain("last 200 lines or 4.0KB"); + await tools.execute({ id: "1", name: "exec", arguments: { command: "ls" } }); + expect(seen).toEqual([{ maxLines: 200, maxBytes: 4096 }]); + }); + + it("prefers `execOutput` to the deprecated shell limits", async () => { + const workspace = makeWorkspace(); + fakeBackends(workspace, [{ id: "sh", callable: false }]); + const tools = createPiTools({ + workspace, + shell: { backends: { sh: {} }, maxLines: 10 }, + execOutput: { maxLines: 50 }, + }); - expect(seen[0].input).toBeNull(); - expect(seen[1].input).toBeUndefined(); + expect(declaration(tools, "exec").description).toContain("last 50 lines"); }); it("applies a schema default when the model omits the field", async () => { @@ -302,7 +415,7 @@ describe("createPiTools execution", () => { arguments: { path: "/workspace/out.png" }, }); - expect(result).toEqual({ + expect(result).toMatchObject({ content: [{ type: "text", text: "bucket unavailable" }], isError: true, }); @@ -329,5 +442,6 @@ describe("createPiTools execution", () => { expect(result.isError).toBe(false); expect(result.content[0]).toMatchObject({ type: "text" }); expect(result.content[1]).toMatchObject({ type: "image", mimeType: "image/png" }); + expect(result).not.toHaveProperty("details"); }); }); diff --git a/packages/computer/src/tools/pi-ai/index.ts b/packages/computer/src/tools/pi-ai/index.ts index 8006a6d3..c119c122 100644 --- a/packages/computer/src/tools/pi-ai/index.ts +++ b/packages/computer/src/tools/pi-ai/index.ts @@ -39,6 +39,7 @@ interface PiToolEntry { description: string; inputSchema: z.ZodType; strictArguments?: boolean; + prepareArguments?: (args: unknown) => unknown; execute: (input: never, context: ToolCallContext) => Promise | AsyncIterable; toModelOutput?: (args: { input: never; output: never }) => ModelOutput; } @@ -73,6 +74,7 @@ export type PiToolResultContent = export interface PiToolResult { content: PiToolResultContent[]; isError: boolean; + details?: unknown; } export interface CreatePiToolsResult { @@ -160,6 +162,7 @@ function piToolEntries(options: CreateToolsOptions): PiToolEntry[] { name: "exec", description: exec.description, inputSchema: exec.inputSchema, + prepareArguments: exec.prepareArguments, execute: (input, context) => exec.execute(input, context), } as PiToolEntry); } @@ -224,7 +227,10 @@ function dispatcher( ); } - const args = dropPlaceholderNulls(call.arguments ?? {}, nullable.get(entry.name) ?? EMPTY); + const prepared = entry.prepareArguments + ? entry.prepareArguments(call.arguments ?? {}) + : (call.arguments ?? {}); + const args = dropPlaceholderNulls(prepared, nullable.get(entry.name) ?? EMPTY); const parsed = entry.inputSchema.safeParse(args); if (!parsed.success) { return errorResult(`Invalid arguments for ${call.name}: ${formatZodError(parsed.error)}`); @@ -239,9 +245,11 @@ function dispatcher( | undefined; try { const output = await settle(run(parsed.data, context)); - return toPiResult( - toOutput ? toOutput({ input: parsed.data, output }) : defaultModelOutput(output), - ); + const modelOutput = toOutput + ? toOutput({ input: parsed.data, output }) + : defaultModelOutput(output); + const result = toPiResult(modelOutput); + return modelOutput.type === "media" ? result : { ...result, details: output }; } catch (err) { return errorResult(err instanceof Error ? err.message : String(err)); } diff --git a/packages/computer/src/workspace.ts b/packages/computer/src/workspace.ts index 12820e6b..3179014b 100644 --- a/packages/computer/src/workspace.ts +++ b/packages/computer/src/workspace.ts @@ -1024,6 +1024,7 @@ export class Workspace { git: this.#gitFactory ? this.git : DISABLED_GIT_CLIENT, artifacts: this.#artifacts, runtime: this.runtime, + ...(this.#assets === undefined ? {} : { assets: this.#assets }), }), ) .then(async (handle) => { diff --git a/packages/dofs/src/fs/find.test.ts b/packages/dofs/src/fs/find.test.ts index af377ab2..90c3c8e9 100644 --- a/packages/dofs/src/fs/find.test.ts +++ b/packages/dofs/src/fs/find.test.ts @@ -223,6 +223,61 @@ describe("find", () => { }); }); + it("prunes the directory a trailing /** names, not only what is below it", async () => { + await withDB(async (db) => { + mkdir(db, "/a/node_modules/pkg", { recursive: true }, () => 0); + mkdir(db, "/a/src", { recursive: true }, () => 0); + await writeFile(db, "/a/src/x.ts", "", {}, () => 0); + await writeFile(db, "/a/node_modules/pkg/index.ts", "", {}, () => 0); + + const modulesInode = resolveInode(db, "/a/node_modules")?.inode; + expect(modulesInode).toBeDefined(); + const listedParents: unknown[] = []; + const all = db.all.bind(db); + // biome-ignore lint/suspicious/noExplicitAny: test spy over the generic method + (db as any).all = (query: string, ...bindings: unknown[]) => { + if (query.includes("FROM vfs_dirents d")) listedParents.push(bindings[0]); + return all(query, ...bindings); + }; + let paths: string[]; + try { + paths = find(db, "/a", undefined, { exclude: ["node_modules/**"] }) + .map((e) => e.path) + .sort(); + } finally { + // biome-ignore lint/suspicious/noExplicitAny: restore the spied method + (db as any).all = all; + } + + expect(paths).toEqual(["/a/src", "/a/src/x.ts"]); + expect(listedParents).not.toContain(modulesInode); + }); + }); + + it("keeps a file a trailing /** names", async () => { + await withDB(async (db) => { + mkdir(db, "/a/sub/.git", { recursive: true }, () => 0); + await writeFile(db, "/a/.git", "gitdir: ../.git/modules/a", {}, () => 0); + await writeFile(db, "/a/sub/.git/HEAD", "", {}, () => 0); + const paths = find(db, "/a", undefined, { exclude: [".git/**", "sub/.git/**"] }) + .map((e) => e.path) + .sort(); + expect(paths).toEqual(["/a/.git", "/a/sub"]); + }); + }); + + it("prunes a directory a **/name/** exclusion names at any depth", async () => { + await withDB(async (db) => { + mkdir(db, "/a/pkg/.git/objects", { recursive: true }, () => 0); + await writeFile(db, "/a/pkg/index.ts", "", {}, () => 0); + await writeFile(db, "/a/pkg/.git/HEAD", "", {}, () => 0); + const paths = find(db, "/a", undefined, { exclude: ["**/.git/**"] }) + .map((e) => e.path) + .sort(); + expect(paths).toEqual(["/a/pkg", "/a/pkg/index.ts"]); + }); + }); + it("applies limit and offset to the surviving matches", async () => { await withDB(async (db) => { mkdir(db, "/a/skip", { recursive: true }, () => 0); @@ -248,6 +303,45 @@ describe("find", () => { }); }); + it("matches any of the alternatives in braces", async () => { + await withDB(async (db) => { + mkdir(db, "/a/src/lib", { recursive: true }, () => 0); + await writeFile(db, "/a/src/x.ts", "", {}, () => 0); + await writeFile(db, "/a/src/lib/y.tsx", "", {}, () => 0); + await writeFile(db, "/a/z.json", "", {}, () => 0); + await writeFile(db, "/a/w.md", "", {}, () => 0); + const paths = find(db, "/a", "{**/*.{ts,tsx},*.json}") + .map((e) => e.path) + .sort(); + expect(paths).toEqual(["/a/src/lib/y.tsx", "/a/src/x.ts", "/a/z.json"]); + }); + }); + + it("treats braces without a comma, or unbalanced, as literal text", async () => { + await withDB(async (db) => { + mkdir(db, "/a", {}, () => 0); + await writeFile(db, "/a/{x}.ts", "", {}, () => 0); + await writeFile(db, "/a/{y.ts", "", {}, () => 0); + await writeFile(db, "/a/x.ts", "", {}, () => 0); + expect(find(db, "/a", "{x}.ts").map((e) => e.path)).toEqual(["/a/{x}.ts"]); + expect(find(db, "/a", "{y.ts").map((e) => e.path)).toEqual(["/a/{y.ts"]); + }); + }); + + it("excludes any of the alternatives in braces", async () => { + await withDB(async (db) => { + mkdir(db, "/a/node_modules", { recursive: true }, () => 0); + mkdir(db, "/a/dist", { recursive: true }, () => 0); + await writeFile(db, "/a/x.ts", "", {}, () => 0); + await writeFile(db, "/a/node_modules/m.ts", "", {}, () => 0); + await writeFile(db, "/a/dist/d.ts", "", {}, () => 0); + const paths = find(db, "/a", "**/*.ts", { exclude: ["{node_modules,dist}/**"] }).map( + (e) => e.path, + ); + expect(paths).toEqual(["/a/x.ts"]); + }); + }); + it("escapes regex metacharacters in literal segments of a pattern", async () => { await withDB(async (db) => { mkdir(db, "/a", {}, () => 0); diff --git a/packages/dofs/src/fs/find.ts b/packages/dofs/src/fs/find.ts index 0e11b270..1450832c 100644 --- a/packages/dofs/src/fs/find.ts +++ b/packages/dofs/src/fs/find.ts @@ -33,7 +33,12 @@ interface WalkStart { path: string; prefix: string; regex: RegExp | undefined; - excludes: RegExp[]; + excludes: Exclusion[]; +} + +interface Exclusion { + regex: RegExp; + directoryOnly: boolean; } const CHILD_PAGE_SIZE = 128; @@ -99,7 +104,13 @@ function prepareWalk( // An empty exclusion pattern is dropped rather than compiled: like // the inclusion glob it would only match the empty relative path, // which no candidate ever has. - const excludes = (exclude ?? []).filter((glob) => glob !== "").map(compileGlob); + const excludes = (exclude ?? []) + .filter((glob) => glob !== "") + .flatMap((glob): Exclusion[] => { + const own = { regex: compileGlob(glob), directoryOnly: false }; + if (!glob.endsWith("/**") || glob.length <= 3) return [own]; + return [own, { regex: compileGlob(glob.slice(0, -3)), directoryOnly: true }]; + }); return { inode: node.inode, path: canonical, @@ -127,7 +138,12 @@ function* walk( // Exclusion is decided before inclusion, and before any child // query: an excluded directory takes its whole subtree with it, // so the walker never reads below it. - if (excludes.some((excluded) => excluded.test(relativePath))) { + if ( + excludes.some( + (excluded) => + (!excluded.directoryOnly || child.type === "dir") && excluded.regex.test(relativePath), + ) + ) { continue; } if (regex === undefined || regex.test(relativePath)) { @@ -161,13 +177,27 @@ function readChildren(db: Database, parentInode: number, afterName: string): Chi // * matches any run of characters except '/' // ** matches any run of characters including '/' // ? matches one character except '/' +// {a,b} matches either alternative; groups nest, and a brace with +// no top-level comma or no closing brace is a literal // Anything else is a literal. Regex metacharacters in literals are // escaped so '.' in '*.ts' doesn't match an arbitrary character. function compileGlob(pattern: string): RegExp { + return new RegExp(`^${globSource(pattern)}$`); +} + +function globSource(pattern: string): string { let re = ""; let i = 0; while (i < pattern.length) { const ch = pattern[i]; + if (ch === "{") { + const alternatives = braceAlternatives(pattern, i); + if (alternatives !== undefined) { + re += `(?:${alternatives.parts.map(globSource).join("|")})`; + i = alternatives.end + 1; + continue; + } + } if (ch === "*") { if (pattern[i + 1] === "*") { // '**/' matches zero or more path segments. Without the slash, '**' @@ -197,7 +227,31 @@ function compileGlob(pattern: string): RegExp { } i += 1; } - return new RegExp(`^${re}$`); + return re; +} + +function braceAlternatives( + pattern: string, + open: number, +): { parts: string[]; end: number } | undefined { + const parts: string[] = []; + let depth = 0; + let start = open + 1; + for (let i = open + 1; i < pattern.length; i += 1) { + const ch = pattern[i]; + if (ch === "{") depth += 1; + else if (ch === "}") { + if (depth === 0) { + parts.push(pattern.slice(start, i)); + return parts.length > 1 ? { parts, end: i } : undefined; + } + depth -= 1; + } else if (ch === "," && depth === 0) { + parts.push(pattern.slice(start, i)); + start = i + 1; + } + } + return undefined; } const REGEX_METACHARS = new Set([".", "+", "?", "^", "$", "(", ")", "[", "]", "{", "}", "|", "\\"]);