From 32c7ddab3e960bb108db744d4f628260cd19a26d Mon Sep 17 00:00:00 2001 From: Ben Gittins <5601410+ihavespoons@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:09:39 +1000 Subject: [PATCH 1/3] Optimizer phase 1: batch settings, the agent's tune request, batched-bench The design is in docs/design/optimizer.md: a measured search for faster llama.cpp settings, with an optional LLM advisor that only proposes. - Model settings batch, ubatch and split_mode, with their llama-server flags, the shared form, the TUI and the console. Old --batch-size and --ubatch-size raw args move to the typed fields on load. - The host agent's tune. request runs llama-bench, llama-batched-bench or a llama-server load check in a backend image, as the benchmark does. Every argument must match a pattern for its flag, so a container cannot pass a model path, a network flag or a device. - The hip, vulkan and cuda images ship llama-batched-bench. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- containers/llama-swap-cuda/Containerfile | 2 +- containers/llama-swap/Containerfile.hip | 4 +- containers/llama-swap/Containerfile.vulkan | 4 +- docs/design/optimizer.md | 104 +++++++++++ docs/index.md | 1 + docs/reference/configuration.md | 3 + docs/reference/system-api.md | 2 +- internal/console/models.go | 1 + internal/console/templates/model.html | 6 + internal/hostctl/agent_test.go | 194 +++++++++++++++++++++ internal/hostctl/hostctl.go | 7 +- internal/llamaswap/generate.go | 12 +- internal/llamaswap/generate_test.go | 19 ++ internal/modelform/modelform.go | 21 +++ internal/models/models.go | 64 +++++++ internal/models/models_test.go | 25 ++- internal/models/tuning.go | 1 + internal/tui/formsections.go | 2 +- internal/tui/models.go | 6 + os/rootfs/usr/libexec/sibyl/agent | 185 ++++++++++++++++++-- 21 files changed, 643 insertions(+), 22 deletions(-) create mode 100644 docs/design/optimizer.md create mode 100644 internal/hostctl/agent_test.go diff --git a/AGENTS.md b/AGENTS.md index 829d8e8..44d6329 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ Sibyl is a Fedora bootc host image plus the container images that it runs. Read - `os/rootfs/` is copied to `/` of the host. Quadlet units are in `os/rootfs/etc/containers/systemd/users/1001/`, and they run rootless as the `sibyl` user (UID 1001). - `containers/` holds one directory per service image. `containers/system-api` builds from the repo root. - `cmd/` and `internal/` are the Go System API. The design is in the "System API and host agent" section of `docs/design/overview.md`, and every route is in `docs/reference/system-api.md`. -- `os/rootfs/usr/libexec/sibyl/agent` is the host agent. It is the only path from a container to systemd. Keep its allowlist in step with `hostctl.Units`. +- `os/rootfs/usr/libexec/sibyl/agent` is the host agent. It is the only path from a container to systemd and Podman. Keep its allowlist in step with `hostctl.Units`. Its `tune` request checks every optimizer argument against a pattern for each flag, and `internal/hostctl/agent_test.go` tests the refusals. - `cmd/sibyl-hostd` is the root agent for hostname, network and accounts. `os/Containerfile` builds it into the host image. Its checks in `internal/hostd` are the security boundary, so keep every new action in `hostd.Decode` and test the refusals. The one exception is `password-check` in `internal/hostd/auth.go`. It is kept out of `Decode` on purpose, so the disk queue refuses it, and only the memory queue `/run/sibyl-auth/req` takes it. - `internal/modelform` is the model settings form that the TUI and the console share. - `internal/console` is the web console at `/console/`. It calls the System API through `internal/client` in process. Keep every console change on an existing API route, and `docs/design/web-console.md` in step. diff --git a/containers/llama-swap-cuda/Containerfile b/containers/llama-swap-cuda/Containerfile index e2b2a80..b0022e6 100644 --- a/containers/llama-swap-cuda/Containerfile +++ b/containers/llama-swap-cuda/Containerfile @@ -22,7 +22,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # The upstream image keeps llama.cpp in /app, where llama-swap lives in the # other Sibyl images. Move it to /opt/llama, so every image has one layout. RUN mkdir -p /opt/llama/bin /opt/llama/lib \ - && mv /app/llama-server /app/llama-bench /opt/llama/bin/ \ + && mv /app/llama-server /app/llama-bench /app/llama-batched-bench /opt/llama/bin/ \ && mv /app/*.so* /opt/llama/lib/ \ && rm -rf /app \ && mkdir -p /app/config \ diff --git a/containers/llama-swap/Containerfile.hip b/containers/llama-swap/Containerfile.hip index f6f82eb..3ccabe4 100644 --- a/containers/llama-swap/Containerfile.hip +++ b/containers/llama-swap/Containerfile.hip @@ -29,12 +29,12 @@ RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \ -DLLAMA_BUILD_EXAMPLES=OFF \ -DLLAMA_BUILD_UI=OFF \ -DLLAMA_USE_PREBUILT_UI=OFF \ - && cmake --build build --config Release -j"$(nproc)" --target llama-server llama-bench + && cmake --build build --config Release -j"$(nproc)" --target llama-server llama-bench llama-batched-bench # Collect the binaries, their shared libraries, and only the rocBLAS and # hipBLASLt kernel files for GPU_TARGETS. The full ROCm tree is ~20 GB. COPY collect-rocm-libs.sh /usr/local/bin/collect-rocm-libs.sh -RUN /usr/local/bin/collect-rocm-libs.sh /out "${GPU_TARGETS}" build/bin/llama-server build/bin/llama-bench +RUN /usr/local/bin/collect-rocm-libs.sh /out "${GPU_TARGETS}" build/bin/llama-server build/bin/llama-bench build/bin/llama-batched-bench FROM docker.io/ubuntu:24.04 AS llama-swap-dl diff --git a/containers/llama-swap/Containerfile.vulkan b/containers/llama-swap/Containerfile.vulkan index 879acaf..053f486 100644 --- a/containers/llama-swap/Containerfile.vulkan +++ b/containers/llama-swap/Containerfile.vulkan @@ -21,9 +21,9 @@ RUN cmake -S . -B build \ -DLLAMA_BUILD_EXAMPLES=OFF \ -DLLAMA_BUILD_UI=OFF \ -DLLAMA_USE_PREBUILT_UI=OFF \ - && cmake --build build --config Release -j"$(nproc)" --target llama-server llama-bench \ + && cmake --build build --config Release -j"$(nproc)" --target llama-server llama-bench llama-batched-bench \ && mkdir -p /out/opt/llama/bin /out/opt/llama/lib \ - && cp build/bin/llama-server build/bin/llama-bench /out/opt/llama/bin/ \ + && cp build/bin/llama-server build/bin/llama-bench build/bin/llama-batched-bench /out/opt/llama/bin/ \ && find build -name '*.so*' -exec cp -P {} /out/opt/llama/lib/ \; FROM docker.io/ubuntu:26.04 AS llama-swap-dl diff --git a/docs/design/optimizer.md b/docs/design/optimizer.md new file mode 100644 index 0000000..413a4bd --- /dev/null +++ b/docs/design/optimizer.md @@ -0,0 +1,104 @@ +# Inference optimizer design + +This document describes the inference optimizer. It finds the llama.cpp settings that run a model fastest on the host's own hardware, by measurement. An upstream LLM can suggest more settings to try, but only a measurement decides. [The design overview](overview.md) holds the rest of the design. + +## Goals + +- Find faster settings for one model on this host, with numbers that the owner can check. +- Never apply settings that were not measured, and never apply settings that do not load at the model's real context size. +- Keep the owner in control: the optimizer recommends, and the owner applies with one action. +- Send no prompts, no user data and no model files off the box. The optional advisor gets hardware facts, model facts and benchmark numbers only. + +## What already exists + +- `internal/models/fit.go` chooses the context size and the KV cache types from the free VRAM. +- `POST /models/{id}/benchmark` runs `llama-bench` once in each backend image through the host agent (`os/rootfs/usr/libexec/sibyl/agent`, `bench`), and recommends the backend with the fastest generation. +- `models.Meta` reads the layer count, the expert count and the expert tensor sizes from the GGUF file. + +The optimizer replaces the fixed benchmark with a search over more settings, and it keeps the same path to the host. + +## Profiles + +The owner picks a profile for each run. + +| Profile | Score | Tool | +|---|---|---| +| `single` | Generation tokens per second at a context depth, then prompt tokens per second | `llama-bench` | +| `throughput` | Total tokens per second over parallel requests, with a floor on the speed of each request | `llama-batched-bench` | + +The context depth of the `single` profile is the number of tokens that are already in the KV cache when the test runs. It is 8,192 tokens, or half the model's context if that is smaller. An empty cache makes generation look faster than a real chat. The floor of the `throughput` profile is half of the single-request generation speed, so a high total with slow answers does not win. + +## Settings that the search covers + +| Setting | llama.cpp flag | Model setting | Note | +|---|---|---|---| +| Batch and micro-batch | `-b`, `-ub` | `batch`, `ubatch` (new) | Large effect on prompt speed. A larger micro-batch needs a larger compute buffer. | +| KV cache type | `-ctk`, `-ctv` | `kv_cache` | `q8_0` and `q4_0` free VRAM for context, and lower quality a little. | +| MoE experts in system RAM | `-ncmoe`, `-ot` | `moe.cpu_layers`, `args` | Only for a model whose experts do not fit in VRAM. | +| Split mode | `-sm` | `split_mode` (new) | `layer` or `row`, only with more than one GPU. | +| Tensor split | `-ts` | `tensor_split` | Only with more than one GPU. | +| Parallel slots | not in `llama-bench` | `parallel` | The `throughput` profile only. | +| Backend | the image | the hardware configuration | The backend applies to every model, so the optimizer reports it apart, and applying it restarts llama-swap. | + +Flash attention stays on, and all layers stay on the GPUs, as llama-swap runs them today. Threads matter only for layers on the CPU, so the search leaves them alone. + +## The search + +The search is a fixed sequence of stages. Each stage runs one tool once. `llama-bench` takes a comma list for each flag and tests every combination, so one run covers one stage. + +1. Baseline: the model's current settings. +2. Backend: each backend that can run on the selected GPUs. With more than one GPU, split mode `layer` and `row`. +3. KV cache: `f16` and `q8_0`. A cache type that does not fit the model's context in VRAM (`FitContext`) is left out. +4. Batch: micro-batch 256, 512, 1024 and 2048, with batch 2048 and 4096, where the batch is at least the micro-batch. +5. MoE: for a model whose experts do not fit in VRAM, the smallest `cpu_layers` value that fits, and two values above it. +6. Advisor: the candidates that the advisor proposes, if the owner turned it on. +7. Parallel: for the `throughput` profile, 1, 2, 4 and 8 parallel sequences at the model's context. +8. Verify: the host agent starts `llama-server` with the best settings at the model's real context size, waits for `/health`, and stops it. + +Each stage keeps the best result so far and passes it to the next stage. A result wins only when it beats the current best by more than three standard deviations of the two runs, or by 3 percent, whichever is larger. `llama-bench` reports the standard deviation (`stddev_ts`). Noise therefore does not change a setting. + +`llama-bench` allocates a context only for the test, not for the model's full context. A micro-batch that fits in the test can still fail at the real context. The verify stage catches that case. If the verify fails, the optimizer drops the last changed setting and verifies again. + +## The path to the host + +The System API runs in a container and has no access to Podman. The optimizer uses the host agent, as the benchmark does. + +- A new agent request, `tune.`, runs one tool from `$RUN/tune/.*` files: the tool (`bench`, `batched` or `server`), the backend, the model path, the GPUs and the arguments. +- The agent checks every argument against an allowlist of flags. Each flag has a pattern for its value. An argument that is not on the list fails the request. The model path, the pool and the GPUs get the same checks as the benchmark. +- The tool runs in the backend image with no network, all capabilities dropped and `no-new-privileges`, as the benchmark does. +- The images build `llama-batched-bench` as well as `llama-server` and `llama-bench`. + +The search itself runs in the System API (`internal/optimize`). It writes one request for each stage, and reads the result files. + +## The advisor + +The advisor is off by default. When the owner turns it on, the optimizer asks one provider model that the router already has. + +- A new router route, `POST /sibyl/advise`, takes a provider and a model. Only the System API key can call it, as with `/sibyl/classify`. The provider keys stay in the router. +- The optimizer sends the GPU names and VRAM, the system RAM, the backend, the model architecture, the parameter count, the quantization, the layer and expert counts, the context size, and every measured result. +- The advisor answers with JSON: up to six candidates, each one a set of the settings in the table above. The optimizer checks each candidate with the same allowlist as the agent, and drops what fails. +- The advisor is most useful for `-ot` patterns on a new MoE architecture, and for the order of the stages on hardware that the fixed search does not know. + +## API + +| Route | Purpose | +|---|---| +| `POST /models/{id}/optimize` | Start a run: `{"profile": "single" or "throughput", "advisor": {"provider": "...", "model": "..."}}`. The advisor is optional. | +| `GET /models/{id}/optimize` | The state: the stage, every measured result, and the recommendation when the run is done. | +| `DELETE /models/{id}/optimize` | Cancel the run. | +| `POST /models/{id}/optimize/apply` | Write the recommended model settings, and reload llama-swap. | + +One run goes at a time on the host, because each run unloads the models and uses every selected GPU. + +## Limits + +- A run unloads the running models, so inference stops for its length: about 5 to 15 minutes for one model. +- `llama-bench` measures the model alone. It does not measure speculative decoding, so a model with a draft model keeps its speculative settings, and the optimizer measures without them. +- A recommendation is for the hardware and the llama.cpp version of the run. An update of either one makes the result old, and the console says so. + +## Phases + +1. Model settings `batch`, `ubatch` and `split_mode`. The `tune` agent request and its allowlist. `llama-batched-bench` in the images. +2. `internal/optimize`, the API routes, and the verify stage. +3. The TUI and the console. +4. The advisor and `/sibyl/advise`. diff --git a/docs/index.md b/docs/index.md index d400114..8ca3832 100644 --- a/docs/index.md +++ b/docs/index.md @@ -39,6 +39,7 @@ These pages are for contributors. They explain how Sibyl works and why. | [Overview](design/overview.md) | The decisions, the host image, the services, storage, security and the phase plan | | [Router](design/router.md) | The model router, the classifier, calibration and their measurements | | [Web console](design/web-console.md) | The web console, its sign-in and its security | +| [Inference optimizer](design/optimizer.md) | The measured search for faster llama.cpp settings, and the optional LLM advisor | | [Diagnostics](design/diagnostics.md) | The watchdog, metrics, incidents, opt-in upload and the fleet service plan | | [Development](development.md) | Building, testing, CI, releases and testing on hardware | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index de0477c..569b5c7 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -61,6 +61,9 @@ qwen3-32b-q4_k_m: | `reasoning.value` | string | `on` or `off` for `enable_thinking` and `thinking`. For `reasoning_effort`, 1 to 32 letters, digits, `_` and `-`. | `--chat-template-kwargs` | The default reasoning. See [Reasoning settings](#reasoning-settings). | | `parallel` | integer | 0 to 64 | `--parallel` | The number of requests that the model answers at once. The requests share the context. 0 leaves llama-server's default. | | `tensor_split` | list of numbers | 0 or more each, at most one value per selected GPU | `--tensor-split` | The share of the layers on each GPU of a spanning model. Empty lets Sibyl split by the memory that each GPU has left. | +| `batch` | integer | 0, or 32 to 65536 | `--batch-size` | The logical batch size. 0 leaves llama-server's default. | +| `ubatch` | integer | 0, or 32 to 65536, and not larger than `batch` | `--ubatch-size` | The physical batch size. A larger value makes the prompt faster and needs a larger compute buffer on the GPU. 0 leaves llama-server's default. | +| `split_mode` | string | `layer` or `row` | `--split-mode` | How a spanning model splits across the GPUs. Empty is `layer`. It has no effect on one GPU. | | `kv_cache.type_k` | string | `f32`, `f16`, `bf16`, `q8_0`, `q4_0`, `q4_1`, `iq4_nl`, `q5_0`, `q5_1` | `--cache-type-k` | The K cache type. | | `kv_cache.type_v` | string | same as `type_k` | `--cache-type-v` | The V cache type. | | `speculative.types` | list of strings | `none`, `draft-simple`, `draft-eagle3`, `draft-mtp`, `draft-dflash`, `draft-dspark`, `ngram-simple`, `ngram-map-k`, `ngram-map-k4v`, `ngram-mod`, `ngram-cache` | `--spec-type` | The speculative decoding types, tried in order. | diff --git a/docs/reference/system-api.md b/docs/reference/system-api.md index eca199a..9e2f9f6 100644 --- a/docs/reference/system-api.md +++ b/docs/reference/system-api.md @@ -159,7 +159,7 @@ Each model in `GET /models` has these fields: | `drafter` | `true` for a DFlash or EAGLE-3 drafter. llama-swap, the router and the Ollama list do not serve it, and it has no `fit`. | | `drafters` | The number of drafters on Hugging Face, from the last search. Missing until a search finishes. `GET /models` starts the searches in the background. | -The settings body of `PUT /models/{id}` has these fields: `layout`, `ctx_size`, `embedding`, `hidden`, `sampling`, `speculative`, `kv_cache`, `moe`, `fit`, `preset`, `reasoning`, `parallel`, `tensor_split` and `args`. [Configuration](configuration.md) describes each field and its range. The route returns `404` for a model that is not installed. It returns `422` in these cases: +The settings body of `PUT /models/{id}` has these fields: `layout`, `ctx_size`, `embedding`, `hidden`, `sampling`, `speculative`, `kv_cache`, `moe`, `fit`, `preset`, `reasoning`, `parallel`, `tensor_split`, `batch`, `ubatch`, `split_mode` and `args`. [Configuration](configuration.md) describes each field and its range. The route returns `404` for a model that is not installed. It returns `422` in these cases: - A value is out of range. - `moe` is set, but the model file has no experts. diff --git a/internal/console/models.go b/internal/console/models.go index 3822de6..0c26066 100644 --- a/internal/console/models.go +++ b/internal/console/models.go @@ -194,6 +194,7 @@ func formFromPost(r *http.Request) modelform.Form { v := func(name string) string { return strings.TrimSpace(r.PostFormValue(name)) } return modelform.Form{ Layout: v("layout"), CtxSize: v("ctxsize"), Parallel: v("parallel"), TensorSplit: v("tensorsplit"), + Batch: v("batch"), UBatch: v("ubatch"), SplitMode: v("splitmode"), Embedding: r.PostFormValue("embedding") == "on", Hidden: r.PostFormValue("hidden") == "on", Temperature: v("temperature"), TopK: v("topk"), TopP: v("topp"), MinP: v("minp"), RepeatPenalty: v("repeatpenalty"), PresencePenalty: v("presencepenalty"), diff --git a/internal/console/templates/model.html b/internal/console/templates/model.html index 6ac9e55..a59e258 100644 --- a/internal/console/templates/model.html +++ b/internal/console/templates/model.html @@ -21,6 +21,12 @@

{{.Model.ID}}

Parallel slots share the context. The tensor split is the share of the layers on each GPU, such as 3,2.

+
+ + + +
+

A larger micro-batch makes the prompt faster and uses more GPU memory. The split mode applies only to a model that spans GPUs.

diff --git a/internal/hostctl/agent_test.go b/internal/hostctl/agent_test.go new file mode 100644 index 0000000..393ba9a --- /dev/null +++ b/internal/hostctl/agent_test.go @@ -0,0 +1,194 @@ +package hostctl + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// checkTuneArgs runs the host agent's allowlist on one argument list. +func checkTuneArgs(t *testing.T, tool string, args ...string) error { + t.Helper() + f := filepath.Join(t.TempDir(), "args") + body := strings.Join(args, "\n") + if len(args) > 0 { + body += "\n" + } + if err := os.WriteFile(f, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return exec.Command("bash", "../../os/rootfs/usr/libexec/sibyl/agent", "--check-tune-args", tool, f).Run() +} + +func TestAgentTuneAllowlist(t *testing.T) { + if _, err := exec.LookPath("bash"); err != nil { + t.Skip("no bash") + } + good := []struct { + tool string + args []string + }{ + {"bench", []string{"-p", "512", "-n", "128", "-d", "8192", "-r", "2"}}, + {"bench", []string{"-b", "2048,4096", "-ub", "256,512,1024,2048"}}, + {"bench", []string{"-ctk", "f16,q8_0", "-ctv", "f16,q8_0", "-sm", "layer,row", "-ts", "3/2,1/1"}}, + {"bench", []string{"-ncmoe", "0,8,16", "-ot", `blk\.(1[0-9])\.ffn_.*_exps\.=CPU;blk\.2.\.ffn_up_exps\.=CPU`}}, + {"batched", []string{"-c", "32768", "-npp", "512", "-ntg", "128", "-npl", "1,2,4,8", "-ctk", "q8_0"}}, + {"server", []string{"-c", "65536", "-b", "4096", "-ub", "1024", "-np", "4", "-sm", "row", "-ts", "3/2"}}, + {"server", nil}, + } + for _, c := range good { + if err := checkTuneArgs(t, c.tool, c.args...); err != nil { + t.Errorf("%s %v refused: %v", c.tool, c.args, err) + } + } + bad := []struct { + tool string + args []string + }{ + {"bench", []string{"-m", "/etc/shadow"}}, // the agent sets the model + {"bench", []string{"--model", "x.gguf"}}, + {"bench", []string{"-p"}}, // a flag with no value + {"bench", []string{"-p", "512", "-n"}}, + {"bench", []string{"-p", "-n"}}, // a flag as a value + {"bench", []string{"-p", "512;rm -rf /"}}, + {"bench", []string{"-p", "1234567"}}, // too large + {"bench", []string{"-ctk", "f32"}}, + {"bench", []string{"-sm", "tensor"}}, + {"bench", []string{"-ot", "-rpc=CPU"}}, // a pattern that starts with a dash + {"bench", []string{"-ot", "blk.*=ROCm0"}}, // only system RAM + {"bench", []string{"-rpc", "10.0.0.1:50052"}}, + {"bench", []string{"-o", "md"}}, + {"bench", []string{"--hf-repo", "evil/model"}}, + {"batched", []string{"-npl", "1,2", "-c", "1,2"}}, // one value for -c + {"batched", []string{"--host", "0.0.0.0"}}, + {"server", []string{"--host", "0.0.0.0"}}, + {"server", []string{"--port", "80"}}, + {"server", []string{"-np", "4,8"}}, + {"server", []string{"--api-key", "x"}}, + {"server", []string{"-c", "4096\n--host"}}, + {"other", []string{"-p", "512"}}, + } + for _, c := range bad { + if err := checkTuneArgs(t, c.tool, c.args...); err == nil { + t.Errorf("%s %q passed", c.tool, c.args) + } + } +} + +func TestTuneRequest(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "requests"), 0o750); err != nil { + t.Fatal(err) + } + c := New(dir) + if err := c.Request(Tune, "qwen3.8-27b-q8_0"); err != nil { + t.Fatal(err) + } + if err := c.Request(Tune, "../x"); err == nil { + t.Error("a bad model ID passed") + } + names, _ := os.ReadDir(filepath.Join(dir, "requests")) + if len(names) != 1 || !strings.HasSuffix(names[0].Name(), ".tune.qwen3.8-27b-q8_0") { + t.Errorf("requests %v", names) + } +} + +// fakeHost lays out the agent's directories, with a podman and a systemctl +// that log their arguments. +type fakeHost struct { + run, models, share, bin, log string +} + +func newFakeHost(t *testing.T) *fakeHost { + t.Helper() + root := t.TempDir() + h := &fakeHost{run: filepath.Join(root, "run"), models: filepath.Join(root, "models"), share: filepath.Join(root, "share"), + bin: filepath.Join(root, "bin"), log: filepath.Join(root, "podman.log")} + for _, d := range []string{filepath.Join(h.run, "requests"), filepath.Join(h.run, "tune"), h.models, h.share, h.bin} { + if err := os.MkdirAll(d, 0o750); err != nil { + t.Fatal(err) + } + } + files := map[string]string{ + filepath.Join(h.models, "m.gguf"): "GGUF", + filepath.Join(h.share, "image-tag"): "sha-test", + filepath.Join(h.share, "gpu-variant"): "amd", + filepath.Join(h.bin, "systemctl"): "#!/bin/sh\necho active\n", + // podman logs each call. llama-bench prints one JSONL row, and a + // started llama-server answers its health check. + filepath.Join(h.bin, "podman"): "#!/bin/sh\necho \"$*\" >> \"" + h.log + "\"\n" + + "case \"$*\" in\n" + + " *llama-bench*) echo '{\"n_prompt\":512,\"avg_ts\":1000.5}' ;;\n" + + " *llama-batched-bench*) echo '{\"pl\":4,\"speed\":80.5}' ;;\n" + + " run\\ -d*) echo ctr ;;\n" + + " inspect*) echo true ;;\n" + + "esac\n", + } + for p, body := range files { + if err := os.WriteFile(p, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + return h +} + +// tune writes a tune request and runs the agent once. +func (h *fakeHost) tune(t *testing.T, tool string, args ...string) (status, out, calls string) { + t.Helper() + files := map[string]string{"path": "m.gguf", "gpus": "0", "pool": "", "backend": "hip", "tool": tool, + "args": strings.Join(args, "\n") + "\n"} + for ext, body := range files { + if err := os.WriteFile(filepath.Join(h.run, "tune", "m."+ext), []byte(body), 0o640); err != nil { + t.Fatal(err) + } + } + _ = os.Remove(h.log) + if err := os.WriteFile(filepath.Join(h.run, "requests", "1.tune.m"), nil, 0o640); err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "../../os/rootfs/usr/libexec/sibyl/agent") + cmd.Env = append(os.Environ(), "PATH="+h.bin+":"+os.Getenv("PATH"), "SIBYL_AGENT_RUN="+h.run, + "SIBYL_AGENT_MODELS="+h.models, "SIBYL_AGENT_POOLS="+filepath.Join(h.run, "nopools"), "SIBYL_AGENT_SHARE="+h.share) + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("agent: %v\n%s", err, b) + } + read := func(p string) string { b, _ := os.ReadFile(p); return string(b) } + return read(filepath.Join(h.run, "tune", "m.status")), read(filepath.Join(h.run, "tune", "m.out")), read(h.log) +} + +func TestAgentTuneRuns(t *testing.T) { + if _, err := exec.LookPath("realpath"); err != nil || exec.Command("realpath", "-e", "/").Run() != nil { + t.Skip("no GNU realpath") + } + h := newFakeHost(t) + status, out, calls := h.tune(t, "bench", "-p", "512", "-n", "128", "-d", "8192", "-ub", "512,1024") + if !strings.HasPrefix(status, "done ") || !strings.Contains(out, `"avg_ts":1000.5`) { + t.Errorf("bench: status %q out %q", status, out) + } + for _, want := range []string{"--network none", "--cap-drop all", "--security-opt no-new-privileges", + "ghcr.io/diffsec/sibyl-llama-swap:sha-test-hip", "-m /models/m.gguf -ngl 999 -fa on -o jsonl -p 512 -n 128 -d 8192 -ub 512,1024", + "HIP_VISIBLE_DEVICES=0", "--device /dev/kfd"} { + if !strings.Contains(calls, want) { + t.Errorf("bench: no %q in\n%s", want, calls) + } + } + + status, out, calls = h.tune(t, "batched", "-c", "32768", "-npl", "1,2,4") + if !strings.HasPrefix(status, "done ") || !strings.Contains(out, `"speed":80.5`) || + !strings.Contains(calls, "--output-format jsonl -c 32768 -npl 1,2,4") { + t.Errorf("batched: status %q out %q calls %s", status, out, calls) + } + + status, out, calls = h.tune(t, "server", "-c", "65536", "-ub", "1024") + if !strings.HasPrefix(status, "done ") || !strings.Contains(out, `"loaded":true`) || + !strings.Contains(calls, "--host 127.0.0.1 --port 8080 -c 65536 -ub 1024") || !strings.Contains(calls, "rm -f sibyl-tune-m") { + t.Errorf("server: status %q out %q calls %s", status, out, calls) + } + + status, _, calls = h.tune(t, "bench", "-m", "/etc/shadow") + if !strings.HasPrefix(status, "failed: the arguments are not allowed") || strings.Contains(calls, "run ") { + t.Errorf("bad args: status %q calls %s", status, calls) + } +} diff --git a/internal/hostctl/hostctl.go b/internal/hostctl/hostctl.go index 40e9349..7fac1e1 100644 --- a/internal/hostctl/hostctl.go +++ b/internal/hostctl/hostctl.go @@ -36,6 +36,9 @@ const ( Reload Action = "reload" // Bench runs llama-bench with both backend images on one model. Bench Action = "bench" + // Tune runs one llama.cpp tool for the optimizer on one model, from the + // files in /tune. The agent checks every argument. + Tune Action = "tune" ) // benchTarget matches the model IDs that the host agent accepts. @@ -74,9 +77,9 @@ func (c *Client) Request(a Action, unit string) error { if !slices.Contains(Units, unit) { return fmt.Errorf("unit %q is not one of %v", unit, Units) } - case Bench: + case Bench, Tune: if !benchTarget.MatchString(unit) { - return fmt.Errorf("model ID %q is not valid for a benchmark", unit) + return fmt.Errorf("model ID %q is not valid for a %s request", unit, a) } default: return fmt.Errorf("unknown action %q", a) diff --git a/internal/llamaswap/generate.go b/internal/llamaswap/generate.go index d2aa943..d7b37cd 100644 --- a/internal/llamaswap/generate.go +++ b/internal/llamaswap/generate.go @@ -193,7 +193,11 @@ func command(m models.Model, s models.Settings, p placement, selected int, devPr for i := range devs { devs[i] = devPrefix + strconv.Itoa(i) } - args = append(args, "--split-mode", "layer", "--device", strings.Join(devs, ",")) + mode := s.SplitMode + if mode == "" { + mode = "layer" + } + args = append(args, "--split-mode", mode, "--device", strings.Join(devs, ",")) if len(s.TensorSplit) > 0 { parts := make([]string, len(s.TensorSplit)) for i, v := range s.TensorSplit { @@ -207,6 +211,12 @@ func command(m models.Model, s models.Settings, p placement, selected int, devPr if s.Parallel > 0 { args = append(args, "--parallel", strconv.Itoa(s.Parallel)) } + if s.Batch > 0 { + args = append(args, "--batch-size", strconv.Itoa(s.Batch)) + } + if s.UBatch > 0 { + args = append(args, "--ubatch-size", strconv.Itoa(s.UBatch)) + } args = append(args, s.KVCache.Flags()...) args = append(args, s.MoE.Flags()...) if !s.Embedding { diff --git a/internal/llamaswap/generate_test.go b/internal/llamaswap/generate_test.go index 8ab43b1..1e62832 100644 --- a/internal/llamaswap/generate_test.go +++ b/internal/llamaswap/generate_test.go @@ -235,6 +235,25 @@ func TestGenerateParallelAndSplit(t *testing.T) { } } +func TestGenerateBatchAndSplitMode(t *testing.T) { + ms := []models.Model{{ID: "big", Path: "big.gguf"}, {ID: "small", Path: "small.gguf"}} + cfg, issues := Generate(twoGPUs(hardware.LayoutSpan), ms, map[string]models.Settings{ + "big": {Batch: 4096, UBatch: 1024, SplitMode: "row"}, + "small": {Layout: "gpu:0", SplitMode: "row"}, + }) + if len(issues) != 0 { + t.Fatalf("issues = %+v", issues) + } + if cmd := cfg.Models["big"].Cmd; !strings.Contains(cmd, "--split-mode row") || !strings.Contains(cmd, "--batch-size 4096") || + !strings.Contains(cmd, "--ubatch-size 1024") { + t.Errorf("big cmd %s", cmd) + } + // One GPU does not split, whatever the split mode says. + if cmd := cfg.Models["small"].Cmd; !strings.Contains(cmd, "--split-mode none") || strings.Contains(cmd, "batch-size") { + t.Errorf("small cmd %s", cmd) + } +} + func TestGenerateMoE(t *testing.T) { cfg, issues := Generate(twoGPUs(hardware.LayoutSpan), installed[:2], map[string]models.Settings{ "big-70b": {MoE: models.MoE{CPULayers: models.AllLayers, CacheMiB: 8192}}, diff --git a/internal/modelform/modelform.go b/internal/modelform/modelform.go index 647c924..c13ab81 100644 --- a/internal/modelform/modelform.go +++ b/internal/modelform/modelform.go @@ -20,6 +20,9 @@ type Form struct { CtxSize string Parallel string TensorSplit string // comma separated + Batch string + UBatch string + SplitMode string // empty (layer), layer or row Embedding bool Hidden bool Temperature string @@ -84,6 +87,7 @@ func FromSettings(s models.Settings) Form { MoELayers: FmtMoELayers(s.MoE.CPULayers), Fit: string(s.Fit), Preset: s.Preset, + SplitMode: s.SplitMode, Args: strings.Join(s.Args, " "), } @@ -99,6 +103,12 @@ func FromSettings(s models.Settings) Form { if s.MoE.CacheMiB > 0 { f.MoECacheMiB = strconv.Itoa(s.MoE.CacheMiB) } + if s.Batch > 0 { + f.Batch = strconv.Itoa(s.Batch) + } + if s.UBatch > 0 { + f.UBatch = strconv.Itoa(s.UBatch) + } split := make([]string, len(s.TensorSplit)) for i, v := range s.TensorSplit { split[i] = strconv.FormatFloat(v, 'f', -1, 64) @@ -168,6 +178,7 @@ func (f Form) Settings() (models.Settings, error) { KVCache: models.KVCache{TypeK: f.KVTypeK, TypeV: f.KVTypeV}, Fit: models.FitMode(f.Fit), Preset: f.Preset, + SplitMode: f.SplitMode, } if v := strings.TrimSpace(f.Reasoning); v != "" { name, value, ok := strings.Cut(v, "=") @@ -191,6 +202,16 @@ func (f Form) Settings() (models.Settings, error) { } else if n != nil { s.Parallel = *n } + if n, e := ParseInt("batch size", f.Batch); e != nil { + return s, e + } else if n != nil { + s.Batch = *n + } + if n, e := ParseInt("micro-batch size", f.UBatch); e != nil { + return s, e + } else if n != nil { + s.UBatch = *n + } if s.MoE.CPULayers, err = ParseMoELayers(f.MoELayers); err != nil { return s, err } diff --git a/internal/models/models.go b/internal/models/models.go index 18c3757..b3804f5 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -200,6 +200,15 @@ type Settings struct { // model (--tensor-split). Empty lets Sibyl split by the memory that // each GPU has left. TensorSplit []float64 `yaml:"tensor_split,omitempty" json:"tensor_split,omitempty"` + // Batch is the logical batch size (--batch-size), and UBatch the + // physical batch size (--ubatch-size). A larger UBatch speeds up the + // prompt and needs a larger compute buffer. 0 leaves llama-server's + // default. + Batch int `yaml:"batch,omitempty" json:"batch,omitempty"` + UBatch int `yaml:"ubatch,omitempty" json:"ubatch,omitempty"` + // SplitMode is how a spanning model splits across the GPUs: layer (the + // default) or row. It has no effect on one GPU. + SplitMode string `yaml:"split_mode,omitempty" json:"split_mode,omitempty"` // Args are extra llama-server flags for anything the typed fields do not cover. Args []string `yaml:"args,omitempty" json:"args,omitempty"` } @@ -207,6 +216,15 @@ type Settings struct { // MaxParallel is the most parallel slots that a model can have. const MaxParallel = 64 +// Limits of the batch sizes. +const ( + MinBatch = 32 + MaxBatch = 65536 +) + +// SplitModes are the split modes that a spanning model can use. +var SplitModes = []string{"layer", "row"} + var ( safeArg = regexp.MustCompile(`^[A-Za-z0-9_.,:=/+-]+$`) reservedArg = map[string]bool{ @@ -249,6 +267,20 @@ func (s Settings) Validate(selected int) error { return errors.New("tensor_split: values must be 0 or more") } } + for _, b := range []struct { + name string + v int + }{{"batch", s.Batch}, {"ubatch", s.UBatch}} { + if b.v != 0 && (b.v < MinBatch || b.v > MaxBatch) { + return fmt.Errorf("%s: must be %d to %d", b.name, MinBatch, MaxBatch) + } + } + if s.Batch > 0 && s.UBatch > s.Batch { + return fmt.Errorf("ubatch %d: must not be larger than batch %d", s.UBatch, s.Batch) + } + if s.SplitMode != "" && !slices.Contains(SplitModes, s.SplitMode) { + return fmt.Errorf("split_mode %q: use one of %s", s.SplitMode, strings.Join(SplitModes, ", ")) + } if err := s.Sampling.Validate(); err != nil { return err } @@ -295,9 +327,41 @@ func LoadSettings(path string) (map[string]Settings, error) { if err := yaml.Unmarshal(data, &out); err != nil { return nil, fmt.Errorf("parse %s: %w", path, err) } + for id, s := range out { + out[id] = s.moveBatchArgs() + } return out, nil } +// batchArgs are the raw args that became the typed Batch and UBatch. +var batchArgs = map[string]func(*Settings) *int{ + "-b": func(s *Settings) *int { return &s.Batch }, "--batch-size": func(s *Settings) *int { return &s.Batch }, + "-ub": func(s *Settings) *int { return &s.UBatch }, "--ubatch-size": func(s *Settings) *int { return &s.UBatch }, +} + +// moveBatchArgs moves batch flags from Args to the typed fields. Before the +// typed fields, the batch sizes could only be set as raw args. +func (s Settings) moveBatchArgs() Settings { + var args []string + for i := 0; i < len(s.Args); i++ { + flag, val, eq := strings.Cut(s.Args[i], "=") + field, ok := batchArgs[flag] + if !ok { + args = append(args, s.Args[i]) + continue + } + if !eq && i+1 < len(s.Args) { + i++ + val = s.Args[i] + } + if n, err := strconv.Atoi(val); err == nil && *field(&s) == 0 { + *field(&s) = n + } + } + s.Args = args + return s +} + // SaveSettings writes models.yaml atomically. func SaveSettings(path string, s map[string]Settings) error { data, err := yaml.Marshal(s) diff --git a/internal/models/models_test.go b/internal/models/models_test.go index df257ce..0caa64a 100644 --- a/internal/models/models_test.go +++ b/internal/models/models_test.go @@ -91,7 +91,12 @@ func TestSettingsValidate(t *testing.T) { {"pin in range", Settings{Layout: "gpu:1"}, false}, {"pin out of range", Settings{Layout: "gpu:2"}, true}, {"bad layout", Settings{Layout: "both"}, true}, - {"safe args", Settings{Args: []string{"--batch-size=2048", "--ubatch-size=512"}}, false}, + {"safe args", Settings{Args: []string{"--threads=8", "--mlock"}}, false}, + {"batch in args", Settings{Args: []string{"--batch-size=2048"}}, true}, + {"batch", Settings{Batch: 4096, UBatch: 1024, SplitMode: "row"}, false}, + {"batch too small", Settings{Batch: 16}, true}, + {"ubatch over batch", Settings{Batch: 512, UBatch: 1024}, true}, + {"bad split mode", Settings{SplitMode: "tensor"}, true}, {"shell chars", Settings{Args: []string{"--temp;rm"}}, true}, {"reserved flag", Settings{Args: []string{"--port=1"}}, true}, {"reserved short flag", Settings{Args: []string{"-m"}}, true}, @@ -125,3 +130,21 @@ func TestSettingsRoundTrip(t *testing.T) { t.Errorf("round trip = %+v", out) } } + +func TestLoadSettingsMovesBatchArgs(t *testing.T) { + p := filepath.Join(t.TempDir(), "models.yaml") + body := "a:\n args: [--batch-size=4096, -ub, \"1024\", --mlock]\nb:\n batch: 2048\n args: [-b, \"512\"]\n" + if err := os.WriteFile(p, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + got, err := LoadSettings(p) + if err != nil { + t.Fatal(err) + } + if a := got["a"]; a.Batch != 4096 || a.UBatch != 1024 || len(a.Args) != 1 || a.Args[0] != "--mlock" { + t.Errorf("a %+v", a) + } + if b := got["b"]; b.Batch != 2048 || len(b.Args) != 0 { + t.Errorf("b keeps its typed batch: %+v", b) + } +} diff --git a/internal/models/tuning.go b/internal/models/tuning.go index 5ada8c4..2786e84 100644 --- a/internal/models/tuning.go +++ b/internal/models/tuning.go @@ -124,6 +124,7 @@ var typedFlags = []string{ "-c", "--ctx-size", "--embedding", "--embeddings", "-np", "--parallel", "-ts", "--tensor-split", "-cmoe", "--cpu-moe", "-ncmoe", "--n-cpu-moe", "--moe-cache-mib", + "-b", "--batch-size", "-ub", "--ubatch-size", } func checkFloat(name string, v *float64, lo, hi float64) error { diff --git a/internal/tui/formsections.go b/internal/tui/formsections.go index 3cf187e..c201ba7 100644 --- a/internal/tui/formsections.go +++ b/internal/tui/formsections.go @@ -24,7 +24,7 @@ type settingsSection struct { } var settingsSections = []settingsSection{ - {"Model", []string{"Layout", "Fit", "Parallel", "CtxSize", "TensorSplit", "KVTypeK", "KVTypeV", "Embedding", "Hidden"}}, + {"Model", []string{"Layout", "Fit", "Parallel", "CtxSize", "TensorSplit", "Batch", "UBatch", "SplitMode", "KVTypeK", "KVTypeV", "Embedding", "Hidden"}}, {"Mixture of Experts", []string{"MoELayers", "MoECacheMiB"}}, {"Sampling defaults", []string{"Preset", "Temperature", "TopK", "TopP", "MinP", "RepeatPenalty", "PresencePenalty"}}, {"Reasoning", []string{"Reasoning"}}, diff --git a/internal/tui/models.go b/internal/tui/models.go index 9137795..d72140a 100644 --- a/internal/tui/models.go +++ b/internal/tui/models.go @@ -200,6 +200,12 @@ func (v *modelsView) editFormWith(m client.Model, fields settingsForm) tea.Cmd { DescriptionFunc(func() string { return contextNote(m, f, hw, gpus) }, f), huh.NewInput().Title("Tensor split").Placeholder(splitHint(m)).Value(&f.TensorSplit). Description("The share of the layers on each GPU, such as 3,2. Empty splits by the memory that each GPU has left."), + optionalNumber("Batch size", "", true, &f.Batch).Placeholder("llama.cpp default"), + optionalNumber("Micro-batch size", "", true, &f.UBatch).Placeholder("llama.cpp default"). + Description("A larger micro-batch makes the prompt faster and uses more GPU memory."), + huh.NewSelect[string]().Title("Split mode").Inline(true).Options( + huh.NewOption("Layer", ""), huh.NewOption("Row", "row"), + ).Value(&f.SplitMode).Description("Only for a model that spans GPUs."), huh.NewSelect[string]().Title("KV cache K type").Inline(true).Options(kv...).Value(&f.KVTypeK), huh.NewSelect[string]().Title("KV cache V type").Inline(true).Options(kv...).Value(&f.KVTypeV), huh.NewConfirm().Title("Embedding model").Value(&f.Embedding), diff --git a/os/rootfs/usr/libexec/sibyl/agent b/os/rootfs/usr/libexec/sibyl/agent index 4559e6a..bd302c2 100755 --- a/os/rootfs/usr/libexec/sibyl/agent +++ b/os/rootfs/usr/libexec/sibyl/agent @@ -10,14 +10,19 @@ # restart. restart one Sibyl unit # reload.all systemctl --user daemon-reload # bench. run llama-bench with each backend image of this variant +# tune. run one llama.cpp tool for the optimizer (see tune) set -euo pipefail -RUN=/var/lib/sibyl/run +# The tests set these. On the host, the systemd unit runs the agent with no +# such variables, and no container can set them. +RUN="${SIBYL_AGENT_RUN:-/var/lib/sibyl/run}" +SHARE="${SIBYL_AGENT_SHARE:-/usr/share/sibyl}" UNITS=(llama-swap router open-webui caddy arcane system-api) REQUEST_RE='^[0-9]{1,20}\.(restart|reload)\.(all|llama-swap|router|open-webui|caddy|arcane|system-api)$' BENCH_RE='^[0-9]{1,20}\.bench\.([a-z0-9][a-z0-9._-]{0,127})$' -MODELS=/var/lib/sibyl/models -POOLS=/var/lib/sibyl-pools +TUNE_RE='^[0-9]{1,20}\.tune\.([a-z0-9][a-z0-9._-]{0,127})$' +MODELS="${SIBYL_AGENT_MODELS:-/var/lib/sibyl/models}" +POOLS="${SIBYL_AGENT_POOLS:-/var/lib/sibyl-pools}" write_status() { local tmp sep="" unit active sub since @@ -65,7 +70,7 @@ bench() { rel="${model#"$(realpath "$root")"/}" split=none [[ "$gpus" == *,* ]] && split=layer - tag="$(cat /usr/share/sibyl/image-tag)" + tag="$(cat "$SHARE/image-tag")" echo "running $(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$dir/$id.status" # Free the GPUs: llama-swap starts the model again on the next request. @@ -74,7 +79,7 @@ bench() { # The backends and GPU devices depend on the image's GPU variant, as in # the System API's drop-in (internal/llamaswap DropIn). local variant backends - variant="$(cat /usr/share/sibyl/gpu-variant)" + variant="$(cat "$SHARE/gpu-variant")" case "$variant" in nvidia) backends="cuda vulkan" ;; intel) backends="vulkan" ;; @@ -101,7 +106,7 @@ bench() { echo "sibyl-agent: benchmarking $id with $backend" if ! podman run --rm --network none \ "${devices[@]}" --group-add keep-groups \ - --userns keep-id:uid=10001,gid=10001 --cap-drop all --security-opt no-new-privileges \ + --userns "keep-id:uid=10001,gid=10001" --cap-drop all --security-opt no-new-privileges \ -v "$root:/models:$mount" -v llama-swap-cache:/app/.cache:U \ -e "$visible=$gpus" --entrypoint /opt/llama/bin/llama-bench "$image" \ -m "/models/$rel" -ngl 999 -fa on -sm "$split" -p 512 -n 128 -r 2 -o json \ @@ -112,9 +117,159 @@ bench() { echo "done $(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$dir/$id.status" } +# The optimizer's arguments. Every flag that a tool takes is here, with a +# pattern for its value. llama-bench takes a comma list for each flag and +# tests every combination. The other tools take one value. +NUM='[0-9]{1,6}' +NUMS="$NUM(,$NUM){0,15}" +KV='(f16|bf16|q8_0|q4_0|q4_1|q5_0|q5_1|iq4_nl)' +SM='(none|layer|row)' +TS='[0-9]{1,3}(\.[0-9]{1,3})?(/[0-9]{1,3}(\.[0-9]{1,3})?){0,15}' +# An override keeps tensors that match a pattern in system RAM. The pattern +# must not start with a dash, so it cannot pass for a flag. +OT='[][A-Za-z0-9_.*+?|()^$\][][A-Za-z0-9_.*+?|()^$\-]{0,199}=CPU' + +# tune_value_re prints the pattern for a flag of a tool, or nothing. +tune_value_re() { + local tool="$1" flag="$2" + case "$tool:$flag" in + bench:-p|bench:-n|bench:-d|bench:-b|bench:-ub|bench:-ncmoe|bench:-r) echo "^$NUMS\$" ;; + bench:-ctk|bench:-ctv) echo "^$KV(,$KV){0,7}\$" ;; + bench:-sm) echo "^$SM(,$SM){0,2}\$" ;; + bench:-ts) echo "^$TS(,$TS){0,7}\$" ;; + bench:-ot) echo "^$OT(;$OT){0,7}(,$OT(;$OT){0,7}){0,7}\$" ;; + batched:-npp|batched:-ntg|batched:-npl) echo "^$NUMS\$" ;; + batched:-c|batched:-b|batched:-ub|batched:-ncmoe|server:-c|server:-b|server:-ub|server:-ncmoe|server:-np) + echo "^$NUM\$" ;; + batched:-ctk|batched:-ctv|server:-ctk|server:-ctv) echo "^$KV\$" ;; + batched:-sm|server:-sm) echo "^$SM\$" ;; + batched:-ts|server:-ts) echo "^$TS\$" ;; + batched:-ot|server:-ot) echo "^$OT(;$OT){0,7}\$" ;; + esac +} + +# tune_check_args checks an argument file: one argument per line, a flag +# then its value. It prints the arguments, one per line, or fails. +tune_check_args() { + local tool="$1" file="$2" flag value re n=0 + local -a lines + mapfile -t lines < "$file" + if (( ${#lines[@]} % 2 != 0 || ${#lines[@]} > 64 )); then + echo "sibyl-agent: tune arguments must be flag and value pairs" >&2 + return 1 + fi + while (( n < ${#lines[@]} )); do + flag="${lines[n]}" value="${lines[n+1]}" + re="$(tune_value_re "$tool" "$flag")" + if [[ -z "$re" || ! "$value" =~ $re ]]; then + echo "sibyl-agent: tune argument '$flag' is not allowed for $tool" >&2 + return 1 + fi + printf '%s\n%s\n' "$flag" "$value" + n=$((n + 2)) + done +} + +tune_fail() { echo "failed: $2" > "$RUN/tune/$1.status"; } + +# tune runs one tool for the optimizer. The API wrote these files in +# $RUN/tune: .path, .pool and .gpus as for bench, .backend, .tool +# (bench, batched or server) and .args. The output goes to .out and +# the tool's log to .log. +tune() { + local id="$1" dir="$RUN/tune" rel gpus pool root mount model backend tool tag image visible variant ctr checked ok=0 i + local -a devices args=() + rel="$(cat "$dir/$id.path" 2>/dev/null || true)" + gpus="$(cat "$dir/$id.gpus" 2>/dev/null || true)" + pool="$(cat "$dir/$id.pool" 2>/dev/null || true)" + backend="$(cat "$dir/$id.backend" 2>/dev/null || true)" + tool="$(cat "$dir/$id.tool" 2>/dev/null || true)" + root="$MODELS" mount=ro,z + if [[ -n "$pool" ]]; then + [[ "$pool" =~ ^pool-[0-9]{1,2}$ ]] || { tune_fail "$id" "invalid tune request"; return; } + root="$POOLS/$pool" mount=ro + fi + model="$(realpath -e -- "$root/$rel" 2>/dev/null || true)" + if [[ -z "$rel" || "$rel" != *.gguf || "$model" != "$(realpath "$root")"/* || ! "$gpus" =~ ^[0-9]+(,[0-9]+)*$ ]]; then + tune_fail "$id" "invalid tune request" + return + fi + rel="${model#"$(realpath "$root")"/}" + variant="$(cat "$SHARE/gpu-variant")" + case "$variant:$backend" in + nvidia:cuda|nvidia:vulkan|intel:vulkan|amd:hip|amd:vulkan) ;; + *) tune_fail "$id" "backend $backend is not in this image"; return ;; + esac + case "$tool" in + bench|batched|server) ;; + *) tune_fail "$id" "unknown tool"; return ;; + esac + if ! checked="$(tune_check_args "$tool" "$dir/$id.args")"; then + tune_fail "$id" "the arguments are not allowed" + return + fi + [[ -n "$checked" ]] && mapfile -t args <<< "$checked" + tag="$(cat "$SHARE/image-tag")" + image="ghcr.io/diffsec/sibyl-llama-swap:${tag}-${backend}" + case "$backend" in + hip) visible=HIP_VISIBLE_DEVICES ;; + cuda) visible=CUDA_VISIBLE_DEVICES ;; + *) visible=GGML_VK_VISIBLE_DEVICES ;; + esac + if [[ "$variant" == nvidia ]]; then + devices=(--device nvidia.com/gpu=all) + elif [[ "$backend" == hip ]]; then + devices=(--device /dev/kfd --device /dev/dri) + else + devices=(--device /dev/dri) + fi + local -a run=(--network none "${devices[@]}" --group-add keep-groups + --userns "keep-id:uid=10001,gid=10001" --cap-drop all --security-opt no-new-privileges + -v "$root:/models:$mount" -v llama-swap-cache:/app/.cache:U -e "$visible=$gpus") + echo "running $(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$dir/$id.status" + rm -f "$dir/$id.out" "$dir/$id.log" + podman exec llama-swap curl -fsS http://127.0.0.1:8080/unload >/dev/null 2>&1 || true + echo "sibyl-agent: tune $id: $tool with $backend" + case "$tool" in + bench) + podman run --rm "${run[@]}" --entrypoint /opt/llama/bin/llama-bench "$image" \ + -m "/models/$rel" -ngl 999 -fa on -o jsonl "${args[@]}" > "$dir/$id.out" 2> "$dir/$id.log" && ok=1 + ;; + batched) + podman run --rm "${run[@]}" --entrypoint /opt/llama/bin/llama-batched-bench "$image" \ + -m "/models/$rel" -ngl 999 -fa on --output-format jsonl "${args[@]}" > "$dir/$id.out" 2> "$dir/$id.log" && ok=1 + ;; + server) + # Start llama-server with the settings, and wait until it has + # loaded the model and allocated the whole context. + ctr="sibyl-tune-$id" + podman rm -f "$ctr" >/dev/null 2>&1 || true + if podman run -d --name "$ctr" "${run[@]}" --entrypoint /opt/llama/bin/llama-server "$image" \ + -m "/models/$rel" -ngl 999 -fa on --host 127.0.0.1 --port 8080 "${args[@]}" >/dev/null 2> "$dir/$id.log"; then + for ((i = 0; i < 600; i++)); do + if podman exec "$ctr" curl -fsS http://127.0.0.1:8080/health >/dev/null 2>&1; then + ok=1 + break + fi + [[ "$(podman inspect -f '{{.State.Running}}' "$ctr" 2>/dev/null)" == true ]] || break + sleep 1 + done + printf '{"loaded":%s,"seconds":%d}\n' "$( ((ok)) && echo true || echo false)" "$i" > "$dir/$id.out" + podman logs "$ctr" >> "$dir/$id.log" 2>&1 || true + podman rm -f "$ctr" >/dev/null 2>&1 || true + fi + ;; + esac + if (( ok )); then + echo "done $(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$dir/$id.status" + else + tune_fail "$id" "$(tail -n 1 "$dir/$id.log" 2>/dev/null | tr -cd '[:print:]' | cut -c1-300)" + fi +} + handle_requests() { local f name action unit reload=0 - local -a restarts=() benches=() + local -a restarts=() benches=() tunes=() shopt -s nullglob for f in "$RUN"/requests/*; do name="$(basename "$f")" @@ -123,6 +278,10 @@ handle_requests() { benches+=("${BASH_REMATCH[1]}") continue fi + if [[ "$name" =~ $TUNE_RE ]]; then + tunes+=("${BASH_REMATCH[1]}") + continue + fi if [[ ! "$name" =~ $REQUEST_RE ]]; then echo "sibyl-agent: rejected request '$name'" >&2 continue @@ -150,9 +309,15 @@ handle_requests() { for unit in "${benches[@]}"; do bench "$unit" done + for unit in "${tunes[@]}"; do + tune "$unit" + done } -if [[ "${1:-}" != "--status" ]]; then - handle_requests -fi +case "${1:-}" in + --status) ;; + # --check-tune-args TOOL FILE checks optimizer arguments, for the tests. + --check-tune-args) tune_check_args "$2" "$3"; exit ;; + *) handle_requests ;; +esac write_status From 6031bd0e1efde2c93518e9ed8e595dfe2aac4ad0 Mon Sep 17 00:00:00 2001 From: Ben Gittins <5601410+ihavespoons@users.noreply.github.com> Date: Sun, 8 Nov 2026 10:16:45 +1000 Subject: [PATCH 2/3] Optimizer phase 2: the search and the API internal/optimize runs the staged search from docs/design/optimizer.md: baseline, backend and split mode, KV cache, batch sizes, MoE layers in system RAM, and parallel requests for the throughput profile. A result wins only when it beats the best by more than 3 percent and 3 standard deviations. The verify stage starts llama-server at the model's real context. When the best does not load, it drops the memory settings one at a time, and tries the measured smaller micro-batches first. The API runs it through the host agent's tune request: POST, GET and DELETE /models/{id}/optimize, and POST /models/{id}/optimize/apply, which reuses the checks of PUT /models/{id}. One run goes at a time. Co-Authored-By: Claude Opus 5.5 --- docs/reference/system-api.md | 4 + internal/api/api.go | 47 +-- internal/api/optimize.go | 365 ++++++++++++++++++++ internal/api/optimize_test.go | 182 ++++++++++ internal/optimize/bench.go | 258 ++++++++++++++ internal/optimize/optimize.go | 517 +++++++++++++++++++++++++++++ internal/optimize/optimize_test.go | 235 +++++++++++++ 7 files changed, 1589 insertions(+), 19 deletions(-) create mode 100644 internal/api/optimize.go create mode 100644 internal/api/optimize_test.go create mode 100644 internal/optimize/bench.go create mode 100644 internal/optimize/optimize.go create mode 100644 internal/optimize/optimize_test.go diff --git a/docs/reference/system-api.md b/docs/reference/system-api.md index 9e2f9f6..dfbcbc4 100644 --- a/docs/reference/system-api.md +++ b/docs/reference/system-api.md @@ -140,6 +140,10 @@ A value that does not fit the variant or the detected GPUs returns `422`. | `DELETE /models/{id}` | None | The apply result | Deletes the model file and all of its shards, removes its settings, and applies. | | `POST /models/{id}/benchmark` | None | `202`, `{"status": "queued"}` | Asks the host agent to run `llama-bench` on the model with each backend image. | | `GET /models/{id}/benchmark` | None | The benchmark state | The progress or the result of the last benchmark. `404` if no benchmark ran. | +| `POST /models/{id}/optimize` | `{"profile": "single"}` or `"throughput"` | `202`, the optimizer state | Starts the optimizer for a chat model. It unloads the models while it runs. `409` while another run goes. | +| `GET /models/{id}/optimize` | None | The optimizer state | The stage, every measured result, and the recommendation when the run is done. `404` if no run took place. | +| `DELETE /models/{id}/optimize` | None | `202` | Cancels the run. The tool that runs on the host finishes first. | +| `POST /models/{id}/optimize/apply` | `{"backend": true}` or `false` | The apply result | Writes the recommended settings. With `backend` true, it also switches the backend of every model when the run found a faster one. `409` if the settings did not load at the model's context. | | `GET /running` | None | The running models | The models that llama-swap runs, the loads that wait, and the last failed load of each model. `502` when llama-swap does not answer. | | `POST /models/{id}/load` | None | `202`, `{"status": "loading"}` | Starts the model in the background, through llama-swap. llama-swap first stops the models that cannot run with it. `409` while the model is loading, `422` for a drafter. | | `POST /models/{id}/unload` | None | `{"status": "unloaded"}` | Stops the model. `422` for a drafter, which stops with its model. | diff --git a/internal/api/api.go b/internal/api/api.go index 08e6e44..75eb825 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -79,6 +79,10 @@ type Server struct { drafters drafterCache drafterTests drafterTests loads loads + optimize optimizeRuns + // tunePoll is how often the optimizer reads the host agent's status. + // Tests shorten it. + tunePoll time.Duration // reloadWait is how long a drafter test waits for llama-swap to read // a new configuration. Tests shorten it. reloadWait time.Duration @@ -150,6 +154,10 @@ func (s *Server) Handler() http.Handler { auth.HandleFunc("DELETE /models/{id}", s.deleteModel) auth.HandleFunc("POST /models/{id}/benchmark", s.startBenchmark) auth.HandleFunc("GET /models/{id}/benchmark", s.getBenchmark) + auth.HandleFunc("POST /models/{id}/optimize", s.startOptimize) + auth.HandleFunc("GET /models/{id}/optimize", s.getOptimize) + auth.HandleFunc("DELETE /models/{id}/optimize", s.cancelOptimize) + auth.HandleFunc("POST /models/{id}/optimize/apply", s.applyOptimize) auth.HandleFunc("GET /models/{id}/drafters", s.getDrafters) auth.HandleFunc("POST /models/{id}/load", s.loadModel) auth.HandleFunc("POST /models/{id}/unload", s.unloadModel) @@ -351,56 +359,57 @@ func (s *Server) putModel(w http.ResponseWriter, r *http.Request) { s.mu.Lock() defer s.mu.Unlock() + if code, err := s.setModel(id, set); err != nil { + writeError(w, code, err) + return + } + s.respondApply(w) +} + +// setModel checks a model's settings and saves them. The caller holds s.mu. +// It returns the HTTP status for an error. +func (s *Server) setModel(id string, set models.Settings) (int, error) { m, err := s.findModel(id) if err != nil { - writeError(w, http.StatusNotFound, err) - return + return http.StatusNotFound, err } // llama.cpp refuses a MoE cache on a dense model, and the model would // not load. meta := s.modelMeta(m) if set.MoE.Enabled() && meta != nil && meta.ExpertCount == 0 { - writeError(w, http.StatusUnprocessableEntity, errors.New("moe: the model file has no experts")) - return + return http.StatusUnprocessableEntity, errors.New("moe: the model file has no experts") } // The variable and the values differ by model family, so the setting // must match what this model's chat template reads. if set.Reasoning != nil && meta != nil { if err := set.Reasoning.CheckAgainst(meta.Reasoning); err != nil { - writeError(w, http.StatusUnprocessableEntity, err) - return + return http.StatusUnprocessableEntity, err } } if d := set.Speculative.DraftModel; d != "" { if d == id { - writeError(w, http.StatusUnprocessableEntity, errors.New("speculative.draft_model: a model cannot be its own draft model")) - return + return http.StatusUnprocessableEntity, errors.New("speculative.draft_model: a model cannot be its own draft model") } if _, err := s.findModel(d); err != nil { - writeError(w, http.StatusUnprocessableEntity, fmt.Errorf("speculative.draft_model: %w", err)) - return + return http.StatusUnprocessableEntity, fmt.Errorf("speculative.draft_model: %w", err) } } hw, err := s.hardware() if err != nil { - writeError(w, http.StatusInternalServerError, err) - return + return http.StatusInternalServerError, err } if err := set.Validate(len(hw.GPUs)); err != nil { - writeError(w, http.StatusUnprocessableEntity, err) - return + return http.StatusUnprocessableEntity, err } settings, err := models.LoadSettings(s.Paths.settingsFile()) if err != nil { - writeError(w, http.StatusInternalServerError, err) - return + return http.StatusInternalServerError, err } settings[id] = set if err := models.SaveSettings(s.Paths.settingsFile(), settings); err != nil { - writeError(w, http.StatusInternalServerError, err) - return + return http.StatusInternalServerError, err } - s.respondApply(w) + return http.StatusOK, nil } func (s *Server) deleteModel(w http.ResponseWriter, r *http.Request) { diff --git a/internal/api/optimize.go b/internal/api/optimize.go new file mode 100644 index 0000000..8a39c18 --- /dev/null +++ b/internal/api/optimize.go @@ -0,0 +1,365 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/diffsec/sibyl/internal/gpu" + "github.com/diffsec/sibyl/internal/hardware" + "github.com/diffsec/sibyl/internal/hostctl" + "github.com/diffsec/sibyl/internal/models" + "github.com/diffsec/sibyl/internal/optimize" +) + +// OptimizeRequest starts an optimizer run. +type OptimizeRequest struct { + Profile optimize.Profile `json:"profile"` +} + +// OptimizeState is the state of a model's last optimizer run. +type OptimizeState struct { + Status string `json:"status"` // running, done, failed or cancelled + Profile optimize.Profile `json:"profile"` + Stage string `json:"stage,omitempty"` + Started string `json:"started"` + Updated string `json:"updated"` + Results []optimize.Result `json:"results,omitempty"` + Recommendation *optimize.Recommendation `json:"recommendation,omitempty"` + Error string `json:"error,omitempty"` + // ImageTag is the backend image version of the run. A recommendation + // is for that llama.cpp build. + ImageTag string `json:"image_tag,omitempty"` + // Applied is set when the owner applied the recommendation. + Applied string `json:"applied,omitempty"` +} + +// OptimizeApply applies a recommendation. Backend also switches the +// backend of every model, when the run found a faster one. +type OptimizeApply struct { + Backend bool `json:"backend"` +} + +// optimizeRuns allows one run at a time: a run unloads the models and uses +// every selected GPU. +type optimizeRuns struct { + mu sync.Mutex + running string + cancel context.CancelFunc +} + +func (s *Server) optimizeDir() string { return filepath.Join(s.Host.Dir, "optimize") } + +func (s *Server) tuneDir() string { return filepath.Join(s.Host.Dir, "tune") } + +func (s *Server) readOptimize(id string) (OptimizeState, bool) { + raw, err := os.ReadFile(filepath.Join(s.optimizeDir(), id+".json")) + if err != nil { + return OptimizeState{}, false + } + var st OptimizeState + if json.Unmarshal(raw, &st) != nil { + return OptimizeState{}, false + } + return st, true +} + +func (s *Server) writeOptimize(id string, st OptimizeState) error { + st.Updated = time.Now().UTC().Format(time.RFC3339) + b, err := json.MarshalIndent(st, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(s.optimizeDir(), 0o750); err != nil { + return err + } + return hardware.WriteFileAtomic(filepath.Join(s.optimizeDir(), id+".json"), b, 0o640) +} + +func (s *Server) startOptimize(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + var req OptimizeRequest + if err := readJSON(r, &req); err != nil { + writeError(w, http.StatusBadRequest, err) + return + } + if req.Profile == "" { + req.Profile = optimize.Single + } + if req.Profile != optimize.Single && req.Profile != optimize.Throughput { + writeError(w, http.StatusUnprocessableEntity, fmt.Errorf("profile %q: use %s or %s", req.Profile, optimize.Single, optimize.Throughput)) + return + } + in, runner, err := s.optimizeInput(id, req.Profile) + if err != nil { + code := http.StatusUnprocessableEntity + if errors.Is(err, os.ErrNotExist) { + code = http.StatusNotFound + } + writeError(w, code, err) + return + } + s.optimize.mu.Lock() + if s.optimize.running != "" { + running := s.optimize.running + s.optimize.mu.Unlock() + writeError(w, http.StatusConflict, fmt.Errorf("the optimizer is running for %s", running)) + return + } + ctx, cancel := context.WithCancel(context.Background()) + s.optimize.running, s.optimize.cancel = id, cancel + s.optimize.mu.Unlock() + + st := OptimizeState{Status: "running", Profile: req.Profile, Stage: "baseline", + Started: time.Now().UTC().Format(time.RFC3339), ImageTag: s.ImageTag} + if err := s.writeOptimize(id, st); err != nil { + s.endOptimize() + writeError(w, http.StatusInternalServerError, err) + return + } + go s.runOptimize(ctx, id, st, in, runner) + writeJSON(w, http.StatusAccepted, st) +} + +func (s *Server) endOptimize() { + s.optimize.mu.Lock() + defer s.optimize.mu.Unlock() + if s.optimize.cancel != nil { + s.optimize.cancel() + } + s.optimize.running, s.optimize.cancel = "", nil +} + +func (s *Server) runOptimize(ctx context.Context, id string, st OptimizeState, in optimize.Input, runner optimize.Runner) { + defer s.endOptimize() + var mu sync.Mutex + search := &optimize.Search{Runner: runner, Progress: func(stage string, results []optimize.Result) { + mu.Lock() + defer mu.Unlock() + st.Stage, st.Results = stage, results + _ = s.writeOptimize(id, st) + }} + rec, err := search.Run(ctx, in) + mu.Lock() + defer mu.Unlock() + switch { + case errors.Is(ctx.Err(), context.Canceled): + st.Status, st.Error = "cancelled", "the owner cancelled the run" + case err != nil: + st.Status, st.Error = "failed", err.Error() + default: + st.Status, st.Stage, st.Recommendation = "done", "done", rec + } + if err := s.writeOptimize(id, st); err != nil { + s.Log.Error("optimizer state", "model", id, "err", err) + } +} + +// optimizeInput describes the model and the host for a run. +func (s *Server) optimizeInput(id string, profile optimize.Profile) (optimize.Input, optimize.Runner, error) { + s.mu.Lock() + defer s.mu.Unlock() + m, err := s.findModel(id) + if err != nil { + return optimize.Input{}, nil, fmt.Errorf("%w: %v", os.ErrNotExist, err) + } + settings, err := models.LoadSettings(s.Paths.settingsFile()) + if err != nil { + return optimize.Input{}, nil, err + } + set := settings[id] + if set.Embedding || m.Drafter { + return optimize.Input{}, nil, errors.New("the optimizer measures chat models, not embedding models or drafters") + } + hw, err := s.hardware() + if err != nil { + return optimize.Input{}, nil, err + } + gpus, _ := gpu.Detect(s.Paths.SysfsRoot) + installed, err := s.scanModels() + if err != nil { + return optimize.Input{}, nil, err + } + _, fits := s.effectiveAll(installed, settings, hw, gpus) + used := hardware.ModelGPUs(set.Layout, false, hw, gpus) + if len(used) == 0 { + return optimize.Input{}, nil, errors.New("no selected GPU can run the model") + } + list := make([]string, len(used)) + for i, g := range used { + list[i] = strconv.Itoa(g.Index) + } + in := optimize.Input{ + Profile: profile, Settings: set, Backend: string(hw.Backend), Backends: s.runnableBackends(hw), GPUs: len(used), + } + meta := s.modelMeta(m) + fit := fits[id] + if fit != nil { + in.Ctx = fit.Ctx + } + if meta != nil { + in.Layers = len(meta.ExpertBytes) + // A KV cache type may not take more memory than the one that the + // model runs with now, so the context still fits. The verify + // stage checks the load at the real context. + if fit != nil { + now := meta.ContextBytes(fit.Ctx, fit.TypeK, fit.TypeV) + in.KVFits = func(k, v string) bool { return meta.ContextBytes(fit.Ctx, k, v) <= now } + } + in.MoE = moeCandidates(set.MoE.CPULayers, in.Layers, meta.ExpertCount) + } + runner := &agentRunner{s: s, id: id, rel: filepath.ToSlash(m.Path), pool: m.Pool, gpus: strings.Join(list, ","), poll: s.tunePoll} + return in, runner, nil +} + +// moeCandidates are the CPUMoE values to try: fewer layers with experts in +// system RAM than now. Fewer is faster, if it loads. +func moeCandidates(now, layers, experts int) []int { + if experts == 0 || now == 0 { + return nil + } + if now == models.AllLayers { + now = layers + } + var out []int + for _, d := range []int{2, 4, 8} { + if n := now - d; n >= 0 { + out = append(out, n) + } + } + return out +} + +// agentRunner runs optimizer tools through the host agent's tune request. +type agentRunner struct { + s *Server + id, rel, pool, gpus string + // poll is the time between status reads. Tests shorten it. + poll time.Duration +} + +// tuneTimeout is the longest one tool run may take. The host agent's +// systemd unit stops a run after 45 minutes. +const tuneTimeout = 45 * time.Minute + +func (a *agentRunner) Run(ctx context.Context, r optimize.Request) ([]byte, error) { + dir := a.s.tuneDir() + if err := os.MkdirAll(dir, 0o750); err != nil { + return nil, err + } + files := map[string]string{ + ".path": a.rel, ".pool": a.pool, ".gpus": a.gpus, ".backend": r.Backend, ".tool": r.Tool, + ".args": strings.Join(r.Args, "\n") + "\n", + ".status": "queued " + time.Now().UTC().Format(time.RFC3339), + } + for ext, body := range files { + if err := hardware.WriteFileAtomic(filepath.Join(dir, a.id+ext), []byte(body), 0o640); err != nil { + return nil, err + } + } + if err := a.s.Host.Request(hostctl.Tune, a.id); err != nil { + return nil, err + } + poll := a.poll + if poll == 0 { + poll = 2 * time.Second + } + deadline := time.Now().Add(tuneTimeout) + for { + raw, _ := os.ReadFile(filepath.Join(dir, a.id+".status")) + status, rest, _ := strings.Cut(strings.TrimSpace(string(raw)), " ") + switch status { + case "done": + return os.ReadFile(filepath.Join(dir, a.id+".out")) + case "failed:": + return nil, fmt.Errorf("%s %s: %s", r.Tool, r.Backend, rest) + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("%s %s: no result after %s", r.Tool, r.Backend, tuneTimeout) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(poll): + } + } +} + +func (s *Server) getOptimize(w http.ResponseWriter, r *http.Request) { + st, ok := s.readOptimize(r.PathValue("id")) + if !ok { + writeError(w, http.StatusNotFound, errors.New("no optimizer run for this model")) + return + } + writeJSON(w, http.StatusOK, st) +} + +func (s *Server) cancelOptimize(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + s.optimize.mu.Lock() + running, cancel := s.optimize.running, s.optimize.cancel + s.optimize.mu.Unlock() + if running != id || cancel == nil { + writeError(w, http.StatusNotFound, errors.New("no optimizer run is going for this model")) + return + } + cancel() + writeJSON(w, http.StatusAccepted, map[string]string{"status": "cancelling"}) +} + +func (s *Server) applyOptimize(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + var req OptimizeApply + if err := readJSON(r, &req); err != nil { + writeError(w, http.StatusBadRequest, err) + return + } + st, ok := s.readOptimize(id) + if !ok || st.Status != "done" || st.Recommendation == nil { + writeError(w, http.StatusNotFound, errors.New("no finished optimizer run for this model")) + return + } + rec := st.Recommendation + if !rec.Verified { + writeError(w, http.StatusConflict, errors.New("the recommendation did not load at the model's context, so it cannot be applied")) + return + } + s.mu.Lock() + defer s.mu.Unlock() + if code, err := s.setModel(id, rec.Settings); err != nil { + writeError(w, code, err) + return + } + if req.Backend && rec.Backend != "" { + hw, err := s.hardware() + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return + } + gpus, err := gpu.Detect(s.Paths.SysfsRoot) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return + } + hw.Backend = hardware.Backend(rec.Backend) + if err := hw.Validate(s.Variant, gpus); err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } + if err := hardware.Save(s.Paths.hardwareFile(), hw); err != nil { + writeError(w, http.StatusInternalServerError, err) + return + } + } + st.Applied = time.Now().UTC().Format(time.RFC3339) + _ = s.writeOptimize(id, st) + s.respondApply(w) +} diff --git a/internal/api/optimize_test.go b/internal/api/optimize_test.go new file mode 100644 index 0000000..818a4cd --- /dev/null +++ b/internal/api/optimize_test.go @@ -0,0 +1,182 @@ +package api + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/diffsec/sibyl/internal/models" + "github.com/diffsec/sibyl/internal/optimize" +) + +// fakeAgent answers tune requests until the test ends: Vulkan generates +// 10 tok/s faster than HIP, and llama-server always loads. hang makes it +// never answer. +func fakeAgent(t *testing.T, f *fixture, hang bool) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { + reqs, tune := filepath.Join(f.run, "requests"), filepath.Join(f.run, "tune") + for ctx.Err() == nil { + time.Sleep(5 * time.Millisecond) + entries, _ := os.ReadDir(reqs) + for _, e := range entries { + _, id, ok := strings.Cut(e.Name()[strings.Index(e.Name(), ".")+1:], "tune.") + if !ok || hang { + continue + } + _ = os.Remove(filepath.Join(reqs, e.Name())) + read := func(ext string) string { + b, _ := os.ReadFile(filepath.Join(tune, id+ext)) + return strings.TrimSpace(string(b)) + } + var out strings.Builder + switch read(".tool") { + case "bench": + v := values(strings.Split(read(".args"), "\n")) + gen := 40.0 + if read(".backend") == "vulkan" { + gen = 50 + } + for _, ub := range v["-ub"] { + for _, ctk := range v["-ctk"] { + for _, ctv := range v["-ctv"] { + for _, b := range v["-b"] { + key := fmt.Sprintf(`"n_batch":%s,"n_ubatch":%s,"type_k":"%s","type_v":"%s","n_cpu_moe":%s,"split_mode":"%s"`, + b, ub, ctk, ctv, v["-ncmoe"][0], v["-sm"][0]) + fmt.Fprintf(&out, "{%s,\"n_prompt\":512,\"n_gen\":0,\"avg_ts\":1000,\"stddev_ts\":1}\n", key) + fmt.Fprintf(&out, "{%s,\"n_prompt\":0,\"n_gen\":128,\"avg_ts\":%v,\"stddev_ts\":0.1}\n", key, gen) + } + } + } + } + case "server": + out.WriteString(`{"loaded":true,"seconds":1}`) + } + _ = os.WriteFile(filepath.Join(tune, id+".out"), []byte(out.String()), 0o640) + _ = os.WriteFile(filepath.Join(tune, id+".status"), []byte("done now"), 0o640) + } + } + }() +} + +func values(args []string) map[string][]string { + m := map[string][]string{} + for i := 0; i+1 < len(args); i += 2 { + m[args[i]] = strings.Split(args[i+1], ",") + } + return m +} + +func waitOptimize(t *testing.T, f *fixture, id, status string) map[string]any { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if code, b := f.do(t, "GET", "/models/"+id+"/optimize", "", true); code == 200 && b["status"] == status { + return b + } + time.Sleep(10 * time.Millisecond) + } + _, b := f.do(t, "GET", "/models/"+id+"/optimize", "", true) + t.Fatalf("no %s state: %v", status, b) + return nil +} + +func TestOptimizeRunAndApply(t *testing.T) { + f := newFixture(t) + f.server.tunePoll = 5 * time.Millisecond + f.do(t, "PUT", "/hardware", `{"backend":"hip","gpus":[1,0],"layout":"span"}`, true) + f.do(t, "PUT", "/models/big", `{"ctx_size":32768,"args":["--mlock"]}`, true) + fakeAgent(t, f, false) + if code, b := f.do(t, "POST", "/models/big/optimize", `{"profile":"single"}`, true); code != http.StatusAccepted || b["status"] != "running" { + t.Fatalf("start = %d %v", code, b) + } + b := waitOptimize(t, f, "big", "done") + raw, _ := json.Marshal(b["recommendation"]) + var rec struct { + Backend string + Verified bool + Settings models.Settings + Changes []map[string]string + } + if err := json.Unmarshal(raw, &rec); err != nil { + t.Fatal(err) + } + if rec.Backend != "vulkan" || !rec.Verified || rec.Settings.CtxSize != 32768 || rec.Settings.Args[0] != "--mlock" { + t.Fatalf("recommendation %s", raw) + } + // The agent got the model and both selected GPUs. + for ext, want := range map[string]string{".path": "big-00001-of-00002.gguf", ".gpus": "0,1"} { + if got, _ := os.ReadFile(filepath.Join(f.run, "tune", "big"+ext)); string(got) != want { + t.Errorf("%s = %q", ext, got) + } + } + if code, b := f.do(t, "POST", "/models/big/optimize/apply", `{"backend":true}`, true); code != 200 { + t.Fatalf("apply = %d %v", code, b) + } + hw, err := f.server.hardware() + if err != nil || hw.Backend != "vulkan" { + t.Errorf("hardware %+v %v", hw, err) + } + set, _ := models.LoadSettings(f.paths.settingsFile()) + if set["big"].CtxSize != 32768 { + t.Errorf("settings %+v", set["big"]) + } + if _, b := f.do(t, "GET", "/models/big/optimize", "", true); b["applied"] == nil { + t.Errorf("applied is not set: %v", b) + } +} + +func TestOptimizeRefusals(t *testing.T) { + f := newFixture(t) + f.server.tunePoll = 5 * time.Millisecond + f.do(t, "PUT", "/hardware", `{"backend":"hip","gpus":[1,0],"layout":"span"}`, true) + for body, want := range map[string]int{`{"profile":"fast"}`: 422, `{"profile":"single","x":1}`: 400} { + if code, _ := f.do(t, "POST", "/models/big/optimize", body, true); code != want { + t.Errorf("%s = %d, want %d", body, code, want) + } + } + if code, _ := f.do(t, "POST", "/models/missing/optimize", `{}`, true); code != http.StatusNotFound { + t.Errorf("missing = %d", code) + } + if code, _ := f.do(t, "GET", "/models/big/optimize", "", true); code != http.StatusNotFound { + t.Errorf("no run = %d", code) + } + if code, _ := f.do(t, "POST", "/models/big/optimize/apply", `{}`, true); code != http.StatusNotFound { + t.Errorf("apply with no run = %d", code) + } + // A run that hangs: a second start conflicts, and cancel ends it. + fakeAgent(t, f, true) + if code, _ := f.do(t, "POST", "/models/big/optimize", `{}`, true); code != http.StatusAccepted { + t.Fatalf("start = %d", code) + } + if code, _ := f.do(t, "POST", "/models/chat-14b/optimize", `{}`, true); code != http.StatusConflict { + t.Errorf("second run = %d", code) + } + if code, _ := f.do(t, "DELETE", "/models/chat-14b/optimize", "", true); code != http.StatusNotFound { + t.Errorf("cancel of another model = %d", code) + } + if code, _ := f.do(t, "DELETE", "/models/big/optimize", "", true); code != http.StatusAccepted { + t.Errorf("cancel = %d", code) + } + waitOptimize(t, f, "big", "cancelled") + // An unverified recommendation cannot be applied. + st, _ := f.server.readOptimize("big") + st.Status = "done" + st.Recommendation = &optimizeRecommendationUnverified + if err := f.server.writeOptimize("big", st); err != nil { + t.Fatal(err) + } + if code, _ := f.do(t, "POST", "/models/big/optimize/apply", `{}`, true); code != http.StatusConflict { + t.Errorf("unverified apply = %d", code) + } +} + +var optimizeRecommendationUnverified = optimize.Recommendation{} diff --git a/internal/optimize/bench.go b/internal/optimize/bench.go new file mode 100644 index 0000000..3003139 --- /dev/null +++ b/internal/optimize/bench.go @@ -0,0 +1,258 @@ +package optimize + +import ( + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/diffsec/sibyl/internal/models" +) + +// kvName is a KV cache type as llama.cpp names it: empty is f16. +func kvName(t string) string { + if t == "" { + return "f16" + } + return t +} + +// commonArgs are the flags of one candidate, in the form that every tool +// takes. +func commonArgs(c Candidate) []string { + var a []string + if c.SplitMode != "" { + a = append(a, "-sm", c.SplitMode) + } else { + a = append(a, "-sm", "none") + } + if c.TensorSplit != "" { + a = append(a, "-ts", c.TensorSplit) + } + if c.Batch > 0 { + a = append(a, "-b", strconv.Itoa(c.Batch)) + } + if c.UBatch > 0 { + a = append(a, "-ub", strconv.Itoa(c.UBatch)) + } + a = append(a, "-ctk", kvName(c.TypeK), "-ctv", kvName(c.TypeV)) + if c.CPUMoE > 0 { + a = append(a, "-ncmoe", strconv.Itoa(c.CPUMoE)) + } + return a +} + +// benchArgs builds one llama-bench run for candidates that share a +// backend. llama-bench tests every combination of the comma lists, so the +// run can measure more than the candidates. parseBench keeps only the +// candidates. +func benchArgs(cands []Candidate, prompt, gen, depth, reps int) []string { + var sm, ts, b, ub, ctk, ctv, moe []string + add := func(list *[]string, v string) { + if !slices.Contains(*list, v) { + *list = append(*list, v) + } + } + for _, c := range cands { + add(&sm, or(c.SplitMode, "none")) + if c.TensorSplit != "" { + add(&ts, c.TensorSplit) + } + add(&b, strconv.Itoa(or0(c.Batch, 2048))) + add(&ub, strconv.Itoa(or0(c.UBatch, 512))) + add(&ctk, kvName(c.TypeK)) + add(&ctv, kvName(c.TypeV)) + add(&moe, strconv.Itoa(c.CPUMoE)) + } + a := []string{"-p", strconv.Itoa(prompt), "-n", strconv.Itoa(gen), "-d", strconv.Itoa(depth), "-r", strconv.Itoa(reps), + "-sm", strings.Join(sm, ","), "-b", strings.Join(b, ","), "-ub", strings.Join(ub, ","), + "-ctk", strings.Join(ctk, ","), "-ctv", strings.Join(ctv, ","), "-ncmoe", strings.Join(moe, ",")} + if len(ts) > 0 { + a = append(a, "-ts", strings.Join(ts, ",")) + } + return a +} + +func or(s, def string) string { + if s == "" { + return def + } + return s +} + +func or0(n, def int) int { + if n == 0 { + return def + } + return n +} + +// benchRow is one line of llama-bench JSONL output. +type benchRow struct { + NBatch int `json:"n_batch"` + NUBatch int `json:"n_ubatch"` + TypeK string `json:"type_k"` + TypeV string `json:"type_v"` + NCPUMoE int `json:"n_cpu_moe"` + SplitMode string `json:"split_mode"` + NPrompt int `json:"n_prompt"` + NGen int `json:"n_gen"` + AvgTS float64 `json:"avg_ts"` + StddevTS float64 `json:"stddev_ts"` +} + +// key is the part of a row that names its settings. The tensor split is +// left out: llama-bench prints it rounded, and one run has one split per +// candidate set. +type benchKey struct { + batch, ubatch int + typeK, typeV string + moe int + split string +} + +func (c Candidate) key() benchKey { + return benchKey{or0(c.Batch, 2048), or0(c.UBatch, 512), kvName(c.TypeK), kvName(c.TypeV), c.CPUMoE, or(c.SplitMode, "none")} +} + +// parseBench reads llama-bench JSONL output, and returns the result of +// each candidate that it measured. +func parseBench(raw []byte, cands []Candidate) ([]Result, error) { + type pair struct{ pp, tg *benchRow } + rows := map[benchKey]*pair{} + sc := bufio.NewScanner(bytes.NewReader(raw)) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + line := bytes.TrimSpace(sc.Bytes()) + if len(line) == 0 || line[0] != '{' { + continue + } + var r benchRow + if err := json.Unmarshal(line, &r); err != nil { + return nil, fmt.Errorf("llama-bench output: %w", err) + } + k := benchKey{r.NBatch, r.NUBatch, r.TypeK, r.TypeV, r.NCPUMoE, r.SplitMode} + p := rows[k] + if p == nil { + p = &pair{} + rows[k] = p + } + switch { + case r.NGen > 0: + p.tg = &r + case r.NPrompt > 0: + p.pp = &r + } + } + if err := sc.Err(); err != nil { + return nil, err + } + var out []Result + for _, c := range cands { + p := rows[c.key()] + if p == nil || p.tg == nil { + continue + } + res := Result{Candidate: c, GenTPS: p.tg.AvgTS, GenStd: p.tg.StddevTS} + if p.pp != nil { + res.PromptTPS, res.PromptStd = p.pp.AvgTS, p.pp.StddevTS + } + out = append(out, res) + } + if len(out) == 0 { + return nil, errors.New("llama-bench measured none of the settings") + } + return out, nil +} + +// batchedRow is one line of llama-batched-bench JSONL output. +type batchedRow struct { + PL int `json:"pl"` + Speed float64 `json:"speed"` + SpeedTG float64 `json:"speed_tg"` +} + +func parseBatched(raw []byte) ([]batchedRow, error) { + var out []batchedRow + for line := range bytes.SplitSeq(raw, []byte("\n")) { + line = bytes.TrimSpace(line) + if len(line) == 0 || line[0] != '{' { + continue + } + var r batchedRow + if err := json.Unmarshal(line, &r); err != nil { + return nil, fmt.Errorf("llama-batched-bench output: %w", err) + } + if r.PL > 0 { + out = append(out, r) + } + } + if len(out) == 0 { + return nil, errors.New("llama-batched-bench printed no results") + } + return out, nil +} + +// fromSettings is the candidate that a model's settings run as. +func fromSettings(s models.Settings, backend string) Candidate { + c := Candidate{Backend: backend, Batch: s.Batch, UBatch: s.UBatch, TypeK: s.KVCache.TypeK, TypeV: s.KVCache.TypeV, + SplitMode: s.SplitMode, Parallel: s.Parallel} + if c.TypeK == "f16" { + c.TypeK = "" + } + if c.TypeV == "f16" { + c.TypeV = "" + } + if s.MoE.CPULayers > 0 { + c.CPUMoE = s.MoE.CPULayers + } + if len(s.TensorSplit) > 0 { + parts := make([]string, len(s.TensorSplit)) + for i, v := range s.TensorSplit { + parts[i] = strconv.FormatFloat(v, 'f', -1, 64) + } + c.TensorSplit = strings.Join(parts, "/") + } + return c +} + +// apply writes the best candidate into the model's settings, and lists +// what changed from the baseline. +func apply(s models.Settings, from, to Candidate) (models.Settings, []Change) { + var ch []Change + set := func(name, a, b string) { + if a != b { + ch = append(ch, Change{Setting: name, From: a, To: b}) + } + } + num := func(n int) string { + if n == 0 { + return "default" + } + return strconv.Itoa(n) + } + set("batch", num(from.Batch), num(to.Batch)) + set("ubatch", num(from.UBatch), num(to.UBatch)) + set("kv_cache.type_k", kvName(from.TypeK), kvName(to.TypeK)) + set("kv_cache.type_v", kvName(from.TypeV), kvName(to.TypeV)) + set("split_mode", or(from.SplitMode, "layer"), or(to.SplitMode, "layer")) + set("moe.cpu_layers", num(from.CPUMoE), num(to.CPUMoE)) + set("parallel", num(from.Parallel), num(to.Parallel)) + s.Batch, s.UBatch = to.Batch, to.UBatch + s.KVCache.TypeK, s.KVCache.TypeV = to.TypeK, to.TypeV + if to.SplitMode == "layer" { + to.SplitMode = "" + } + s.SplitMode = to.SplitMode + if s.MoE.CPULayers != models.AllLayers || to.CPUMoE != from.CPUMoE { + s.MoE.CPULayers = to.CPUMoE + } + if to.Parallel > 0 { + s.Parallel = to.Parallel + } + return s, ch +} diff --git a/internal/optimize/optimize.go b/internal/optimize/optimize.go new file mode 100644 index 0000000..9ce2833 --- /dev/null +++ b/internal/optimize/optimize.go @@ -0,0 +1,517 @@ +// Package optimize searches llama.cpp settings for the fastest run of one +// model on this host, by measurement. The design is in +// docs/design/optimizer.md. +// +// The search is a fixed sequence of stages. Each stage asks the host agent +// to run one tool once (Runner), and keeps the best result so far. A +// result wins only when it beats the best by more than the noise of the +// measurement. The last stage starts llama-server with the best settings at +// the model's real context, so a recommendation always loads. +package optimize + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strconv" + "strings" + "time" + + "github.com/diffsec/sibyl/internal/models" +) + +// Profile is what the search scores. +type Profile string + +// The profiles. +const ( + // Single scores generation speed at a context depth, then prompt speed. + Single Profile = "single" + // Throughput scores the total speed of parallel requests, with a floor + // on the speed of each request. + Throughput Profile = "throughput" +) + +// Tools that the host agent runs. +const ( + ToolBench = "bench" + ToolBatched = "batched" + ToolServer = "server" +) + +// Request is one tool run on the host. +type Request struct { + Tool string + Backend string + Args []string +} + +// Runner runs a tool on the host and returns its standard output. +type Runner interface { + Run(ctx context.Context, r Request) ([]byte, error) +} + +// Candidate is one set of settings that the search measures. +type Candidate struct { + Backend string `json:"backend"` + SplitMode string `json:"split_mode,omitempty"` // layer or row, with more than one GPU + TensorSplit string `json:"tensor_split,omitempty"` // llama-bench form, such as 3/2 + Batch int `json:"batch,omitempty"` + UBatch int `json:"ubatch,omitempty"` + TypeK string `json:"type_k,omitempty"` // empty is f16 + TypeV string `json:"type_v,omitempty"` + CPUMoE int `json:"cpu_moe,omitempty"` // layers with experts in system RAM + Parallel int `json:"parallel,omitempty"` +} + +// Result is the measurement of one candidate. +type Result struct { + Candidate + Stage string `json:"stage"` + PromptTPS float64 `json:"prompt_tps,omitempty"` + PromptStd float64 `json:"prompt_std,omitempty"` + GenTPS float64 `json:"gen_tps,omitempty"` + GenStd float64 `json:"gen_std,omitempty"` + // TotalTPS and RequestTPS are the throughput profile's numbers at + // Parallel requests: all tokens per second, and the generation speed + // of one request. + TotalTPS float64 `json:"total_tps,omitempty"` + RequestTPS float64 `json:"request_tps,omitempty"` + Error string `json:"error,omitempty"` +} + +// Input describes the model and the host for one run. +type Input struct { + Profile Profile + // Settings are the model's settings now. The recommendation starts + // from them. + Settings models.Settings + // Backend is the backend that llama-swap runs now, and Backends every + // backend that can run on the selected GPUs. + Backend string + Backends []string + // GPUs is the number of GPUs that the model runs on. + GPUs int + // Ctx is the context size that the model runs with. + Ctx int + // KVFits tells whether a KV cache type keeps the context in VRAM. + KVFits func(typeK, typeV string) bool + // MoE are the values of CPUMoE to try. Empty skips the stage. + MoE []int + // Layers is the model's block count. A model with every expert in + // system RAM (models.AllLayers) runs with CPUMoE at Layers. + Layers int + // Proposals are extra candidates, from the advisor. + Proposals []Candidate +} + +// Change is one setting that the recommendation changes. +type Change struct { + Setting string `json:"setting"` + From string `json:"from"` + To string `json:"to"` +} + +// Recommendation is the outcome of a run. +type Recommendation struct { + Baseline Result `json:"baseline"` + Best Result `json:"best"` + // Settings are the model's new settings. Backend is set when a + // different backend was faster. The backend applies to every model. + Settings models.Settings `json:"settings"` + Backend string `json:"backend,omitempty"` + Changes []Change `json:"changes"` + // Verified is true when llama-server loaded the settings at the + // model's context. + Verified bool `json:"verified"` +} + +// Progress reports the stage and every result so far. +type Progress func(stage string, results []Result) + +// Search runs the stages. +type Search struct { + Runner Runner + Progress Progress + // Repetitions of each llama-bench test. The noise margin needs two. + Repetitions int + results []Result + in Input +} + +// Default test sizes. +const ( + promptTokens = 512 + genTokens = 128 + maxDepth = 8192 +) + +// depth is the number of tokens in the KV cache when a test runs. +func (in Input) depth() int { + if in.Ctx > 0 { + return min(maxDepth, in.Ctx/2) + } + return 4096 +} + +// Run searches, and returns the recommendation. +func (s *Search) Run(ctx context.Context, in Input) (*Recommendation, error) { + s.in, s.results = in, nil + if s.Repetitions < 2 { + s.Repetitions = 3 + } + if in.Profile != Single && in.Profile != Throughput { + return nil, fmt.Errorf("profile %q: use %s or %s", in.Profile, Single, Throughput) + } + cur := fromSettings(in.Settings, in.Backend) + if in.Settings.MoE.CPULayers == models.AllLayers { + cur.CPUMoE = in.Layers + } + base, err := s.bench(ctx, "baseline", []Candidate{cur}) + if err != nil { + return nil, err + } + if len(base) == 0 { + return nil, errors.New("the baseline measurement failed: " + s.lastError()) + } + baseline := base[0] + best := baseline + pick := func(stage string, cands []Candidate) error { + rs, err := s.bench(ctx, stage, cands) + if err != nil { + return err + } + for _, r := range rs { + if better(r, best) { + best = r + } + } + return nil + } + // Backend and split mode. + var cands []Candidate + for _, b := range in.Backends { + for _, sm := range s.splitModes() { + c := best.Candidate + c.Backend, c.SplitMode = b, sm + if c != best.Candidate { + cands = append(cands, c) + } + } + } + if err := pick("backend", cands); err != nil { + return nil, err + } + // KV cache. + cands = nil + for _, t := range []string{"", "q8_0"} { + for _, v := range []string{"", "q8_0"} { + if in.KVFits != nil && !in.KVFits(t, v) { + continue + } + c := best.Candidate + c.TypeK, c.TypeV = t, v + if c != best.Candidate { + cands = append(cands, c) + } + } + } + if err := pick("kv-cache", cands); err != nil { + return nil, err + } + // Batch sizes. + cands = nil + for _, b := range []int{2048, 4096} { + for _, ub := range []int{256, 512, 1024, 2048} { + c := best.Candidate + c.Batch, c.UBatch = b, ub + if c != best.Candidate { + cands = append(cands, c) + } + } + } + if err := pick("batch", cands); err != nil { + return nil, err + } + // MoE experts in system RAM. + cands = nil + for _, n := range in.MoE { + c := best.Candidate + c.CPUMoE = n + if c != best.Candidate { + cands = append(cands, c) + } + } + if err := pick("moe", cands); err != nil { + return nil, err + } + // The advisor's proposals. + if err := pick("advisor", in.Proposals); err != nil { + return nil, err + } + if in.Profile == Throughput { + if best, err = s.parallel(ctx, best); err != nil { + return nil, err + } + } + rec := &Recommendation{Baseline: baseline} + rec.Best, rec.Verified, err = s.verify(ctx, best, baseline) + if err != nil { + return nil, err + } + rec.Settings, rec.Changes = apply(in.Settings, baseline.Candidate, rec.Best.Candidate) + if rec.Best.Backend != in.Backend { + rec.Backend = rec.Best.Backend + rec.Changes = append(rec.Changes, Change{Setting: "backend", From: in.Backend, To: rec.Best.Backend}) + } + s.report("done") + return rec, nil +} + +func (s *Search) splitModes() []string { + if s.in.GPUs > 1 { + return []string{"layer", "row"} + } + return []string{""} +} + +func (s *Search) report(stage string) { + if s.Progress != nil { + s.Progress(stage, slices.Clone(s.results)) + } +} + +func (s *Search) lastError() string { + for i := len(s.results) - 1; i >= 0; i-- { + if e := s.results[i].Error; e != "" { + return e + } + } + return "no result" +} + +// better tells whether a beats b by more than the noise. Generation speed +// decides first. When it is level, prompt speed decides. +func better(a, b Result) bool { + if a.Error != "" { + return false + } + if a.TotalTPS > 0 || b.TotalTPS > 0 { + return a.TotalTPS > b.TotalTPS*1.03 + } + gen := max(3*b.GenStd, 3*a.GenStd, 0.03*b.GenTPS) + switch { + case a.GenTPS > b.GenTPS+gen: + return true + case a.GenTPS < b.GenTPS-gen: + return false + } + prompt := max(3*b.PromptStd, 3*a.PromptStd, 0.03*b.PromptTPS) + return a.PromptTPS > b.PromptTPS+prompt +} + +// bench measures candidates with llama-bench: one run for each backend, +// with comma lists for the flags that differ. +func (s *Search) bench(ctx context.Context, stage string, cands []Candidate) ([]Result, error) { + if len(cands) == 0 { + return nil, nil + } + s.report(stage) + var out []Result + for _, group := range groupByBackend(cands) { + args := benchArgs(group, promptTokens, genTokens, s.in.depth(), s.Repetitions) + raw, err := s.Runner.Run(ctx, Request{Tool: ToolBench, Backend: group[0].Backend, Args: args}) + if ctx.Err() != nil { + return nil, ctx.Err() + } + var rs []Result + if err == nil { + rs, err = parseBench(raw, group) + } + if err != nil { + for _, c := range group { + s.results = append(s.results, Result{Candidate: c, Stage: stage, Error: err.Error()}) + } + continue + } + for i := range rs { + rs[i].Stage = stage + } + s.results = append(s.results, rs...) + out = append(out, rs...) + } + s.report(stage) + return out, nil +} + +// parallel finds the number of parallel requests with the most total +// speed, where each request still gets half the speed of one alone. +func (s *Search) parallel(ctx context.Context, best Result) (Result, error) { + s.report("parallel") + c := best.Candidate + args := append(commonArgs(c), "-c", strconv.Itoa(max(s.in.Ctx, promptTokens+genTokens)), + "-npp", strconv.Itoa(promptTokens), "-ntg", strconv.Itoa(genTokens), "-npl", "1,2,4,8") + raw, err := s.Runner.Run(ctx, Request{Tool: ToolBatched, Backend: c.Backend, Args: args}) + if ctx.Err() != nil { + return best, ctx.Err() + } + var rows []batchedRow + if err == nil { + rows, err = parseBatched(raw) + } + if err != nil { + s.results = append(s.results, Result{Candidate: c, Stage: "parallel", Error: err.Error()}) + s.report("parallel") + return best, nil + } + floor := best.GenTPS / 2 + out := best + for _, r := range rows { + res := Result{Candidate: c, Stage: "parallel", PromptTPS: best.PromptTPS, GenTPS: best.GenTPS, + TotalTPS: r.Speed, RequestTPS: r.SpeedTG / float64(r.PL)} + res.Parallel = r.PL + s.results = append(s.results, res) + if res.RequestTPS >= floor && (out.TotalTPS == 0 || better(res, out)) { + out = res + } + } + s.report("parallel") + return out, nil +} + +// verify starts llama-server with the best settings at the model's +// context. llama-bench allocates a context only for its test, so a setting +// that it measured can still fail to load at the real context. When the +// best does not load, verify drops the settings that need GPU memory, one +// at a time: the batch sizes, then the KV cache type, then half the +// parallel requests. A variant that was not measured is measured first. At +// last it keeps the baseline. +func (s *Search) verify(ctx context.Context, best, baseline Result) (Result, bool, error) { + s.report("verify") + variants := []Candidate{best.Candidate} + // The measured smaller micro-batches, fastest first. + var smaller []Result + for _, r := range s.results { + q, b := r.Candidate, best.Candidate + q.Batch, q.UBatch, q.Parallel = b.Batch, b.UBatch, b.Parallel + if r.Error == "" && r.Stage != "verify" && q == b && or0(r.UBatch, 512) < or0(best.UBatch, 512) { + smaller = append(smaller, r) + } + } + sort.SliceStable(smaller, func(i, j int) bool { return better(smaller[i], smaller[j]) }) + for _, r := range smaller { + c := best.Candidate + c.Batch, c.UBatch = r.Batch, r.UBatch + variants = append(variants, c) + } + c := best.Candidate + c.Batch, c.UBatch = baseline.Batch, baseline.UBatch + variants = append(variants, c) + c.TypeK, c.TypeV = baseline.TypeK, baseline.TypeV + variants = append(variants, c) + if c.Parallel > 1 { + c.Parallel /= 2 + variants = append(variants, c) + } + variants = append(variants, baseline.Candidate) + tried := map[Candidate]bool{} + for _, v := range variants { + if tried[v] { + continue + } + tried[v] = true + r, ok := s.measured(v) + if !ok { + rs, err := s.bench(ctx, "verify", []Candidate{v}) + if err != nil { + return best, false, err + } + if len(rs) == 0 { + continue + } + r = rs[0] + r.Parallel = v.Parallel + } + if v != baseline.Candidate && better(baseline, r) { + continue // the variant is slower than the settings now + } + loaded, err := s.load(ctx, v) + if err != nil { + return best, false, err + } + if loaded { + s.report("verify") + return r, true, nil + } + } + return baseline, false, nil +} + +// measured finds the result of a candidate that a stage measured. +func (s *Search) measured(c Candidate) (Result, bool) { + for i := len(s.results) - 1; i >= 0; i-- { + if r := s.results[i]; r.Error == "" && r.Stage != "verify" && r.Candidate == c { + return r, true + } + } + // llama-bench does not run parallel requests, so a candidate that + // differs only in Parallel has the same single-request numbers. + p := c + p.Parallel = 0 + for i := len(s.results) - 1; i >= 0; i-- { + if r := s.results[i]; r.Error == "" && r.Stage != "verify" && r.Stage != "parallel" { + q := r.Candidate + q.Parallel = 0 + if q == p { + r.Candidate = c + return r, true + } + } + } + return Result{}, false +} + +// load starts llama-server with a candidate at the model's context. +func (s *Search) load(ctx context.Context, c Candidate) (bool, error) { + args := commonArgs(c) + if s.in.Ctx > 0 { + args = append(args, "-c", strconv.Itoa(s.in.Ctx)) + } + if c.Parallel > 0 { + args = append(args, "-np", strconv.Itoa(c.Parallel)) + } + start := time.Now() + raw, err := s.Runner.Run(ctx, Request{Tool: ToolServer, Backend: c.Backend, Args: args}) + if ctx.Err() != nil { + return false, ctx.Err() + } + if err == nil && strings.Contains(string(raw), `"loaded":true`) { + return true, nil + } + msg := "llama-server did not load" + if err != nil { + msg = err.Error() + } + s.results = append(s.results, Result{Candidate: c, Stage: "verify", + Error: fmt.Sprintf("%s after %s", msg, time.Since(start).Round(time.Second))}) + s.report("verify") + return false, nil +} + +// groupByBackend splits candidates by backend, in order. +func groupByBackend(cands []Candidate) [][]Candidate { + var out [][]Candidate + idx := map[string]int{} + for _, c := range cands { + i, ok := idx[c.Backend] + if !ok { + i = len(out) + idx[c.Backend] = i + out = append(out, nil) + } + out[i] = append(out[i], c) + } + return out +} diff --git a/internal/optimize/optimize_test.go b/internal/optimize/optimize_test.go new file mode 100644 index 0000000..e4dc935 --- /dev/null +++ b/internal/optimize/optimize_test.go @@ -0,0 +1,235 @@ +package optimize + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "testing" + + "github.com/diffsec/sibyl/internal/models" +) + +// fakeHost answers tool runs from a model of a GPU: +// - generation: 50 tok/s, +5 on vulkan, +10 with row split, -2 per MoE +// layer in system RAM, -1 with a q8_0 cache (inside the noise); +// - prompt: 2000 tok/s, +600 at ubatch 1024, +700 at 2048; +// - llama-server does not load with ubatch 2048 (the compute buffer). +type fakeHost struct { + calls []Request + fail map[string]bool // tools that fail +} + +func values(args []string) map[string][]string { + m := map[string][]string{} + for i := 0; i+1 < len(args); i += 2 { + m[args[i]] = strings.Split(args[i+1], ",") + } + return m +} + +func (h *fakeHost) gen(backend, sm, ctk string, moe int) float64 { + g := 50.0 + if backend == "vulkan" { + g += 5 + } + if sm == "row" { + g += 10 + } + if ctk == "q8_0" { + g-- + } + return g - 2*float64(moe) +} + +func (h *fakeHost) Run(_ context.Context, r Request) ([]byte, error) { + h.calls = append(h.calls, r) + if h.fail[r.Tool] { + return nil, errors.New(r.Tool + " failed") + } + v := values(r.Args) + var b strings.Builder + switch r.Tool { + case ToolBench: + for _, sm := range v["-sm"] { + for _, bs := range v["-b"] { + for _, ub := range v["-ub"] { + for _, ctk := range v["-ctk"] { + for _, ctv := range v["-ctv"] { + for _, moe := range v["-ncmoe"] { + n, _ := strconv.Atoi(moe) + bn, _ := strconv.Atoi(bs) + ubn, _ := strconv.Atoi(ub) + pp := 2000.0 + switch ubn { + case 1024: + pp += 600 + case 2048: + pp += 700 + } + row := map[string]any{"n_batch": bn, "n_ubatch": ubn, "type_k": ctk, "type_v": ctv, "n_cpu_moe": n, "split_mode": sm} + for _, test := range []struct { + np, ng int + ts float64 + }{{512, 0, pp}, {0, 128, h.gen(r.Backend, sm, ctk, n)}} { + row["n_prompt"], row["n_gen"], row["avg_ts"], row["stddev_ts"] = test.np, test.ng, test.ts, 0.2 + j, _ := json.Marshal(row) + b.Write(j) + b.WriteString("\n") + } + } + } + } + } + } + } + case ToolBatched: + for _, pl := range []int{1, 2, 4, 8} { + // More requests give more total speed, and each one gets slower. + tg := 60 * float64(pl) / (1 + 0.3*float64(pl-1)) + fmt.Fprintf(&b, `{"pl":%d,"speed":%f,"speed_tg":%f}`+"\n", pl, tg*1.2, tg) + } + case ToolServer: + if v["-ub"] != nil && v["-ub"][0] == "2048" { + return []byte(`{"loaded":false,"seconds":3}`), nil + } + return []byte(`{"loaded":true,"seconds":12}`), nil + } + return []byte(b.String()), nil +} + +func TestSearchSingle(t *testing.T) { + h := &fakeHost{} + var stages []string + s := &Search{Runner: h, Progress: func(stage string, _ []Result) { + if len(stages) == 0 || stages[len(stages)-1] != stage { + stages = append(stages, stage) + } + }} + rec, err := s.Run(context.Background(), Input{ + Profile: Single, Settings: models.Settings{CtxSize: 65536, Args: []string{"--mlock"}}, + Backend: "hip", Backends: []string{"hip", "vulkan"}, GPUs: 2, Ctx: 65536, + KVFits: func(k, v string) bool { return true }, + }) + if err != nil { + t.Fatal(err) + } + b := rec.Best + // vulkan with row split is fastest. q8_0 is inside the noise, so it + // stays f16. ubatch 2048 is faster but does not load, so 1024 wins. + if b.Backend != "vulkan" || b.SplitMode != "row" || b.TypeK != "" || b.UBatch != 1024 || !rec.Verified { + t.Errorf("best %+v verified %v", b.Candidate, rec.Verified) + } + if rec.Backend != "vulkan" || rec.Settings.SplitMode != "row" || rec.Settings.UBatch != 1024 || + rec.Settings.CtxSize != 65536 || rec.Settings.Args[0] != "--mlock" { + t.Errorf("recommendation %+v", rec) + } + if rec.Baseline.GenTPS != 50 || rec.Best.GenTPS != 65 { + t.Errorf("baseline %v best %v", rec.Baseline.GenTPS, rec.Best.GenTPS) + } + want := "baseline backend kv-cache batch verify done" + if got := strings.Join(stages, " "); got != want { + t.Errorf("stages %q", got) + } + // Every bench run sets the depth: half the context, at most 8192. + for _, c := range h.calls { + if c.Tool == ToolBench && values(c.Args)["-d"][0] != "8192" { + t.Errorf("depth %v", c.Args) + } + } + // The verify of ubatch 2048 failed, and that is in the results. + var failed bool + for _, r := range s.results { + failed = failed || (r.Stage == "verify" && r.UBatch == 2048 && r.Error != "") + } + if !failed { + t.Error("the failed verify is not in the results") + } +} + +func TestSearchKVFitAndMoE(t *testing.T) { + h := &fakeHost{} + rec, err := (&Search{Runner: h}).Run(context.Background(), Input{ + Profile: Single, Settings: models.Settings{MoE: models.MoE{CPULayers: models.AllLayers}}, + Backend: "hip", Backends: []string{"hip"}, GPUs: 1, Ctx: 8192, Layers: 40, + KVFits: func(k, v string) bool { return k == "q8_0" && v == "q8_0" }, // only q8_0 fits + MoE: []int{10, 12, 14}, + }) + if err != nil { + t.Fatal(err) + } + // The baseline ran with all 40 layers in system RAM. + first := values(h.calls[0].Args) + if first["-ncmoe"][0] != "40" || first["-d"][0] != "4096" || first["-sm"][0] != "none" { + t.Errorf("baseline args %v", h.calls[0].Args) + } + for _, c := range h.calls { + if v := values(c.Args); c.Tool == ToolBench && strings.Contains(strings.Join(v["-ctk"], ","), "f16") && + strings.Contains(strings.Join(v["-ctv"], ","), "q8_0") { + t.Errorf("a KV type that does not fit was measured: %v", c.Args) + } + } + if rec.Best.CPUMoE != 10 || rec.Settings.MoE.CPULayers != 10 { + t.Errorf("best %+v settings %+v", rec.Best.Candidate, rec.Settings.MoE) + } +} + +func TestSearchThroughput(t *testing.T) { + h := &fakeHost{} + rec, err := (&Search{Runner: h}).Run(context.Background(), Input{ + Profile: Throughput, Backend: "hip", Backends: []string{"hip"}, GPUs: 1, Ctx: 32768, + }) + if err != nil { + t.Fatal(err) + } + // Per request at 8 is 60*8/3.1/8 = 19.4, under half of 50. At 4 it is + // 60*4/1.9/4 = 31.6, so 4 wins. + if rec.Best.Parallel != 4 || rec.Settings.Parallel != 4 { + t.Errorf("best %+v", rec.Best) + } + last := h.calls[len(h.calls)-1] + if last.Tool != ToolServer || values(last.Args)["-np"][0] != "4" || values(last.Args)["-c"][0] != "32768" { + t.Errorf("verify %v", last) + } +} + +func TestSearchFailures(t *testing.T) { + // A bench that fails at the baseline stops the run. + if _, err := (&Search{Runner: &fakeHost{fail: map[string]bool{ToolBench: true}}}).Run(context.Background(), + Input{Profile: Single, Backend: "hip"}); err == nil || !strings.Contains(err.Error(), "bench failed") { + t.Errorf("err %v", err) + } + // A server that never loads keeps the baseline, not verified. + rec, err := (&Search{Runner: &fakeHost{fail: map[string]bool{ToolServer: true}}}).Run(context.Background(), + Input{Profile: Single, Backend: "hip", Backends: []string{"hip"}, GPUs: 1, Ctx: 4096}) + if err != nil { + t.Fatal(err) + } + if rec.Verified || len(rec.Changes) != 0 { + t.Errorf("recommendation %+v", rec) + } + if _, err := (&Search{Runner: &fakeHost{}}).Run(context.Background(), Input{Profile: "fast"}); err == nil { + t.Error("a bad profile passed") + } +} + +func TestBetterNoise(t *testing.T) { + base := Result{GenTPS: 50, GenStd: 0.5, PromptTPS: 2000, PromptStd: 10} + for _, c := range []struct { + r Result + want bool + }{ + {Result{GenTPS: 51, PromptTPS: 2000}, false}, // inside 3% + {Result{GenTPS: 52, PromptTPS: 2000}, true}, // over 3% and 3 sigma + {Result{GenTPS: 50, PromptTPS: 2100}, true}, // gen level, prompt 5% better + {Result{GenTPS: 47, PromptTPS: 3000}, false}, // gen worse + {Result{GenTPS: 60, GenStd: 5, PromptTPS: 2000}, false}, // noisy + {Result{GenTPS: 80, PromptTPS: 2000, Error: "oom"}, false}, // failed + } { + if got := better(c.r, base); got != c.want { + t.Errorf("%+v: %v", c.r, got) + } + } +} From a0ebcab80e83fbd0d97e27e7c1d73ed0161f014a Mon Sep 17 00:00:00 2001 From: Ben Gittins <5601410+ihavespoons@users.noreply.github.com> Date: Sun, 8 Nov 2026 11:28:01 +1000 Subject: [PATCH 3/3] Bump the fedora-bootc base image digest Co-Authored-By: Claude Opus 5.5 --- os/Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/os/Containerfile b/os/Containerfile index 1421453..9dda0fb 100644 --- a/os/Containerfile +++ b/os/Containerfile @@ -26,7 +26,7 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/sibyl-hostd ./cmd/ FROM scratch AS variants COPY os/variants/ / -FROM quay.io/fedora/fedora-bootc:44@sha256:3f4b21855f45581d8c82cd44931c2a06e8a13e93d871f0c70fc387fad7e89e01 +FROM quay.io/fedora/fedora-bootc:44@sha256:6bda175e89575aa3dfb6b0b4b8342978cb7de8d6109042f0338560eb3fe82c15 ARG GPU_VARIANT=amd ARG SIBYL_VERSION=dev