diff --git a/containers/llama-swap/Containerfile.hip b/containers/llama-swap/Containerfile.hip index 3ccabe4..a1ec1fd 100644 --- a/containers/llama-swap/Containerfile.hip +++ b/containers/llama-swap/Containerfile.hip @@ -36,6 +36,60 @@ RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \ COPY collect-rocm-libs.sh /usr/local/bin/collect-rocm-libs.sh RUN /usr/local/bin/collect-rocm-libs.sh /out "${GPU_TARGETS}" build/bin/llama-server build/bin/llama-bench build/bin/llama-batched-bench +# Edge0 (edge0-strata) for Radeon RX 7900 cards. The engine is not llama.cpp: +# it runs an Edge0 latent MoE model from a model folder, and its Python server +# gives the OpenAI API. llama-swap starts it through edge0-launch.py. +# +# EDGE0_REF is a commit of github.com/diffsec/Edge0, the fork that carries the +# gfx11 kernels. EDGE0_GPU_TARGETS must stay gfx1100: the kernels and +# the hipBLASLt tuning table are for that architecture only. +FROM docker.io/rocm/dev-ubuntu-24.04:7.2.4-complete@sha256:92f309c51b52cef8762867848f1529dee821624f23cd5df38455e819538f762f AS edge0-build + +ARG EDGE0_REF=49d5def3bf41807acc7eb5b0b07319f75bbcff88 +# The llama.cpp commit that edge0-strata/EDGE0.md names. It gives ggml to the build. +ARG LLAMA_CPP_REF=3cf03257f219afbe7334045ff7c6a06ac68c627d +ARG EDGE0_GPU_TARGETS="gfx1100" +# ROCm 7.2.4 has hipBLASLt 1.2.2, which is tuning table 100202. +ARG HIPBLASLT_TABLE=gfx1100-hipblaslt-100202.txt + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential cmake ninja-build git ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +RUN git init -q /edge0 \ + && git -C /edge0 fetch -q --depth 1 https://github.com/diffsec/Edge0 "${EDGE0_REF}" \ + && git -C /edge0 checkout -q FETCH_HEAD \ + && mkdir -p /edge0/edge0-strata/third_party/llama.cpp \ + && curl -fsSL "https://github.com/ggml-org/llama.cpp/archive/${LLAMA_CPP_REF}.tar.gz" \ + | tar -xz --strip-components=1 -C /edge0/edge0-strata/third_party/llama.cpp \ + && ln -s /edge0/edge0-strata /src + +# STRATA_PREFILL_MMQ=ON is required: without it the link stops with undefined +# symbols such as strata::prefill::fused_q3k::experts_latent_q3k. +WORKDIR /src +RUN HIP_PATH="$(hipconfig -R)" \ + cmake -S . -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DSTRATA_ENABLE_HIP=ON \ + -DSTRATA_ENABLE_CUDA=OFF \ + -DCMAKE_HIP_ARCHITECTURES="${EDGE0_GPU_TARGETS}" \ + -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang++" \ + -DCMAKE_PREFIX_PATH=/opt/rocm \ + -DSTRATA_PREFILL_MMQ=ON \ + -DSTRATA_NATIVE_EXPERTS=ON \ + -DSTRATA_GGML_DIR=/src/third_party/llama.cpp \ + && cmake --build build --target strata -j"$(nproc)" + +# The engine, its shared libraries, and only the rocBLAS and hipBLASLt kernel +# files for GPU_TARGETS. The engine reads the tuning table from tuning/ beside +# its binary. Without the table a 5.3K prompt ran at 725 tok/s, not 1150. +COPY collect-rocm-libs.sh /usr/local/bin/collect-rocm-libs.sh +RUN /usr/local/bin/collect-rocm-libs.sh /out "${EDGE0_GPU_TARGETS}" build/strata \ + && mkdir -p /out/opt/edge0 /out/opt/llama/bin/tuning \ + && cp "tools/hip/${HIPBLASLT_TABLE}" /out/opt/llama/bin/tuning/ \ + && cp -r serve strata_control strata_cli tools requirements.txt /out/opt/edge0/ \ + && find /out/opt/edge0 -name '__pycache__' -prune -exec rm -rf {} + + FROM docker.io/ubuntu:24.04 AS llama-swap-dl ARG LLAMA_SWAP_VERSION=266 @@ -50,13 +104,19 @@ FROM docker.io/ubuntu:24.04 RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates libgomp1 libssl3t64 curl \ - libdrm2 libdrm-amdgpu1 libelf1t64 libnuma1 \ + libdrm2 libdrm-amdgpu1 libelf1t64 libnuma1 python3 python3-venv \ && rm -rf /var/lib/apt/lists/* \ && groupadd --system --gid 10001 app \ && useradd --system --uid 10001 --gid 10001 --home /app --no-create-home app COPY --from=build /out/ / COPY --from=llama-swap-dl /out/llama-swap /app/llama-swap +# The Edge0 engine, its server, and the server's Python packages at the +# versions that the engine pins. +COPY --from=edge0-build /out/ / +RUN python3 -m venv /opt/venv \ + && /opt/venv/bin/pip install --no-cache-dir -r /opt/edge0/requirements.txt +COPY edge0-launch.py /opt/edge0/launch.py COPY config.yaml /app/config/config.yaml COPY check-libs.sh /usr/local/bin/check-libs.sh # Fail the build if any binary or library has an unresolved dependency. @@ -70,7 +130,9 @@ ENV PATH=/app:/opt/llama/bin:${PATH} \ ROCBLAS_TENSILE_LIBPATH=/opt/rocm/lib/rocblas/library \ HIPBLASLT_TENSILE_LIBPATH=/opt/rocm/lib/hipblaslt/library \ GPU_MAX_HW_QUEUES=1 \ - SIBYL_GPU_BACKEND=ROCm + SIBYL_GPU_BACKEND=ROCm \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 USER 10001:10001 WORKDIR /app diff --git a/containers/llama-swap/edge0-launch.py b/containers/llama-swap/edge0-launch.py new file mode 100644 index 0000000..57c597b --- /dev/null +++ b/containers/llama-swap/edge0-launch.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Start edge0-strata's server for one Edge0 model folder. + +llama-swap runs this as the command of an Edge0 model. The folder's +model.json is a complete server configuration for an NVIDIA start from inside +the folder. This script changes what differs here, writes the result to a +work directory, and replaces itself with the server. +""" +import argparse +import json +import os +import sys + +ENGINE = "/opt/llama/bin/strata" +SERVER = "/opt/edge0/serve/server.py" +ROCM_LIB = "/opt/rocm/lib" +# Options whose value differs here, with the number of values each one takes. +DROP = {"--pool-workers": 1, "--max-context": 1, "--vram-reserve-later-mib": 1} +# A card that another model also uses can hold less than the engine planned +# for. On the test host a later card with 240 MiB free stalled the first +# request, and 1400 MiB of reserve did not. +RESERVE_LATER_MIB = "1400" + + +def absolute(value, model_dir): + """Resolve a "./" path of the folder's configuration against the folder.""" + if value == ".": + return model_dir + if isinstance(value, str) and value.startswith("./"): + return os.path.join(model_dir, value[2:]) + return value + + +# On one GPU the engine keeps the expert file in system RAM, plus about +# RAM_OVERHEAD. On a layer split it keeps only the experts that no GPU holds +# (--resident-experts). On two 24 GB cards that was 21.4 GiB of a 37.3 GiB +# file, which is 57%, so SPAN_SHARE leaves a margin. The same figures are in +# internal/models (Edge0RAM). +RAM_OVERHEAD = 6 << 30 +SPAN_SHARE = 0.65 + + +def ram_short(experts_path, span=False, meminfo="/proc/meminfo"): + """Return (needed, available) in bytes when RAM is short, else None.""" + try: + experts = os.path.getsize(experts_path) + need = (int(experts * SPAN_SHARE) if span else experts) + RAM_OVERHEAD + with open(meminfo, encoding="ascii") as f: + for line in f: + if line.startswith("MemAvailable:"): + avail = int(line.split()[1]) << 10 + return (need, avail) if avail < need else None + except (OSError, ValueError, IndexError): + pass + return None # the size or the memory figure cannot be read + + +def configure(manifest, model_dir, alias, gpus, context, work): + """Return the server configuration for this container.""" + args, src, i = [], manifest["args"], 0 + while i < len(src): + arg = src[i] + # The engine picks the worker count from the CPU when the option is absent. + if arg in DROP: + i += 1 + DROP[arg] + continue + # --pcie-mode dma is not tested on a layer split with the engine's + # split fix. Before the fix it stopped the first request. + if arg == "--pcie-mode" and len(gpus) > 1: + i += 2 + continue + args.append(absolute(arg, model_dir)) + i += 1 + args += ["--max-context", str(context)] + if len(gpus) > 1: + args += ["--vram-reserve-later-mib", RESERVE_LATER_MIB] + # Two cards hold most of the experts, so RAM keeps only the rest. + # Without this the engine keeps the whole expert file in RAM too. + if "--resident-experts" not in args: + args.append("--resident-experts") + + env = {k: absolute(v, model_dir) for k, v in manifest.get("env", {}).items()} + return { + "model_name": alias, + "backend": "hip", + "exe": ENGINE, + "args": args, + "cwd": work, + "log": os.path.join(work, "engine.log"), + "tokenizer": absolute(manifest.get("tokenizer", "./tokenizer"), model_dir), + "gpus_asked": True, + "gpu": gpus if len(gpus) > 1 else gpus[0], + "lib_dirs": [ROCM_LIB], + "env": env, + } + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--model-dir", required=True) + ap.add_argument("--alias", required=True, help="the model name that the server answers to") + ap.add_argument("--gpus", required=True, help="GPU numbers as HIP numbers them here, such as 0 or 0,1") + ap.add_argument("--context", required=True, type=int) + ap.add_argument("--port", required=True, type=int) + ap.add_argument("--work", default="/tmp/edge0", help="a writable directory for the configuration and the engine log") + ap.add_argument("--print-config", action="store_true", help="print the configuration and stop") + opt = ap.parse_args() + + gpus = [int(g) for g in opt.gpus.split(",")] + with open(os.path.join(opt.model_dir, "model.json"), encoding="utf-8") as f: + manifest = json.load(f) + work = os.path.join(opt.work, opt.alias) + cfg = configure(manifest, opt.model_dir, opt.alias, gpus, opt.context, work) + if opt.print_config: + json.dump(cfg, sys.stdout, indent=1) + return + # Without the RAM, the engine pushes the host into swap or the kernel + # stops a process. Refuse the load with a message that says why. + short = ram_short(cfg["env"].get("STRATA_LATENT_A", os.path.join(opt.model_dir, "experts.bin")), span=len(gpus) > 1) + if short: + sys.exit("edge0: %s keeps %.1f GiB of system RAM, and %.1f GiB is available" + % (opt.alias, short[0] / (1 << 30), short[1] / (1 << 30))) + # The image sets GPU_MAX_HW_QUEUES=1 for idle llama-server processes. This + # engine waits on the GPU in one stream for a copy in another stream. On + # one hardware queue the copy never starts, and the first request stops + # with "verify: timed out at layer N". + os.environ.pop("GPU_MAX_HW_QUEUES", None) + os.makedirs(work, exist_ok=True) + path = os.path.join(work, "config.json") + with open(path, "w", encoding="utf-8") as f: + json.dump(cfg, f, indent=1) + os.execv(sys.executable, [sys.executable, SERVER, "--engine", "strata", "--config", path, + "--host", "127.0.0.1", "--port", str(opt.port), "--no-api-key"]) + + +if __name__ == "__main__": + main() diff --git a/docs/design/overview.md b/docs/design/overview.md index b437c11..54de2fb 100644 --- a/docs/design/overview.md +++ b/docs/design/overview.md @@ -9,7 +9,7 @@ Sibyl is an all-in-one inference server. It is a hardened, immutable Linux host - Each GPU vendor has its own host image variant: `-amd`, `-nvidia` and `-intel`. - The AMD variant is tested on a host with two Radeon RX 7900 XTX cards (the "test host" in this document). - The NVIDIA and Intel variants build and pass the same checks, but no one has run them on hardware yet. -- Models are GGUF files only, quantized to fit consumer GPUs. Sibyl does not use vLLM. +- Models are GGUF files, quantized to fit consumer GPUs. The one exception is an Edge0 model folder on the AMD variant. Sibyl does not use vLLM. ## Decisions @@ -19,8 +19,9 @@ Sibyl is an all-in-one inference server. It is a hardened, immutable Linux host | Container runtime | Podman with Quadlet, rootless | Fedora default. Quadlet units start at boot without a daemon. | | Container UI | Arcane (BSD-3-Clause) | Compose editor, Git Sync, native Podman and Quadlet pages. Dockhand is source-available (BSL 1.1), so it is out. | | Inference engine | llama.cpp `llama-server` behind llama-swap | One engine for every GPU vendor, with fast single-stream GGUF inference. llama-swap swaps models and places them on GPUs. | +| Second engine | edge0-strata for Edge0 model folders, HIP image only, as a llama-swap model | llama.cpp cannot run the Edge0 latent MoE models. llama-swap starts the engine like any other model, so the swap rules, the router and the load routes do not change. | | GPU backends | HIP (ROCm) and Vulkan for AMD, CUDA and Vulkan for NVIDIA, Vulkan for Intel | The fastest backend differs by GPU, quant and context size. The benchmark compares the variant's backends. | -| Model format | GGUF only | Quantized models fit consumer GPUs. No vLLM. | +| Model format | GGUF, and Edge0 model folders | Quantized models fit consumer GPUs. No vLLM. | | Multi-GPU | Per-model layout through the llama-swap matrix router | One model can span both cards, or two models can run on one card each. | | GPU vendors | Separate image variants: `-amd`, `-nvidia`, `-intel` | Only the NVIDIA variant carries out-of-tree kernel modules, signed with the project MOK key. NVIDIA and Intel are untested on hardware. | | Base for service images | Wolfi where possible | Smaller images with fewer CVEs. The inference images use the vendors' bases, because Wolfi does not package ROCm or CUDA. | @@ -153,6 +154,31 @@ The HIP image sets `GPU_MAX_HW_QUEUES=1`. With the default of 4 hardware queues | `LLAMA_SWAP_VERSION` | pinned release | llama-swap release | | `GPU_TARGETS` | `gfx1030;gfx1100;gfx1101;gfx1102;gfx1150;gfx1151;gfx1200;gfx1201` | RDNA2 (RX 6800 and 6900, PRO W6800), RDNA3 (RX 7000, PRO W7000), RDNA3.5 (Ryzen AI 300, Strix Halo), RDNA4 (RX 9000, AI PRO R9700) | | `GGML_HIP_RCCL` | `OFF` | RCCL helps only `--split-mode tensor`. Upstream disables it by default. | +| `EDGE0_REF` | pinned commit of `diffsec/Edge0` | The Edge0 engine, HIP image only | + +### Edge0 models + +An Edge0 model is a folder, not a GGUF file. The folder holds a `model.json` with `"family": "Edge0"`, an expert file, and the rest of the pack. The scanner lists the folder as one model with `engine` set to `edge0` (`models.Scan`). llama.cpp cannot run it. The engine is edge0-strata, from the fork `github.com/diffsec/Edge0`, which carries the kernels for the Radeon RX 7900 matrix cores. + +The HIP image builds the engine in a stage of its own, for `gfx1100` only, and adds its Python server. `EDGE0_REF` pins the fork's commit. The model's llama-swap command is `containers/llama-swap/edge0-launch.py`. It reads the folder's `model.json`, which is a server configuration for a start from inside the folder, and changes these parts: + +- The paths become absolute, and the work directory and the engine log go to `/tmp/edge0/`. The container's root is read-only. +- `--max-context` comes from the model's `ctx_size`. With no setting it is 131072, the value in `model.json` and the value that the engine was measured with. +- `--pool-workers` is removed. The engine then takes the count from the CPU. +- It removes `GPU_MAX_HW_QUEUES` from the engine's environment. The engine waits on the GPU in one stream for a copy in another stream. On one hardware queue the copy never starts, and the first request stops with `verify: timed out at layer 1`. +- On more than one GPU it adds `--resident-experts`, the engine's low-RAM mode: RAM keeps only the experts that no GPU holds. It also adds `--vram-reserve-later-mib 1400` and removes `--pcie-mode dma`, which is not tested on a layer split. + +The model's layout works as for a GGUF model. A model that spans the selected GPUs runs as a layer split, in a matrix set of its own. A model pinned to one GPU shares the `pinned` set, so a GGUF model on another GPU runs beside it. The other llama-server settings do not apply, and the TUI and the console show only the layout, the context size and the hidden flag. The benchmark and the optimizer refuse an Edge0 model, because both measure llama.cpp. + +On one GPU the engine keeps the whole expert file in system RAM and copies experts from there to the GPU. Sibyl counts the size of `experts.bin` plus 6 GiB as the model's RAM (`models.Edge0RAM`), which is 43.3 GiB for Edge0 145B. On the test host that model took 41 GiB of RAM, beside 22.6 GB of VRAM on one card. A model that spans GPUs keeps less, because the cards hold more of the experts: on two 24 GB cards, 27.0 GiB of the 37.3 GiB of experts were in VRAM, 21.4 GiB stayed in RAM, and the process took 32.5 GB. For a span Sibyl counts 65% of the expert file plus 6 GiB (`Model.Edge0RAMOn`), which is 30.3 GiB for Edge0 145B. `POST /models/{id}/load` refuses the load with `422` when `MemAvailable` in `/proc/meminfo` is lower, unless the model already runs. The launcher makes the same check, because a chat request also starts a load, and llama-swap then answers the request with the launcher's message. + +On the test host, Edge0 145B (Q3_K) on one RX 7900 XTX in this image, with the Quadlet's restrictions, decoded chat answers at 44.1 to 56.1 tokens per second. It read a 5.3K-token prompt at 1029 to 1201 tokens per second and a 21K-token prompt at 1291. The idle card drew 17 W with the model loaded. A layer split across both cards decoded at 32.2 to 51.2 tokens per second, and it read a 5.3K-token prompt at 273 to 285 tokens per second and a 21K-token prompt at 413. One of the test host's cards is on a PCIe 3.0 x1 link, and the prompt path streams experts to it. + +The span needs the fork at `49d5def` or later. Before that commit, the second card of a split was given pointers to the first card's latent weights and token embedding table. A card cannot read another card's VRAM, so the first request never finished. + +The download picker offers an Edge0 repository as one variant, the whole folder (`hf.Client.Edge0Variant`). A repository counts as Edge0 when `model.json` and `experts.bin` are at its root. The variant leaves out the `assets` folder and hidden files, and its fit is `split` when the host's total RAM holds the model, or `no`. The download keeps the subfolders. Hugging Face gives a SHA-256 for LFS files only, so the download checks each small file against its Git blob ID, which is the SHA-1 of `blob `, a zero byte, and the content. The commit pins both kinds of file. + +Only the Q3_K folders run. The NVFP4 folders need kernels that the fork has for CUDA only, and Sibyl does not stop a download of one. The base images are plain `FROM` lines, so Renovate can update them. The HIP build uses `docker.io/rocm/dev-ubuntu-24.04:7.2.4-complete`. Upstream llama.cpp tests ROCm 7.2.x, and ROCm 7.14 is a benchmark candidate. Renovate leaves the ROCm and Ubuntu releases alone, because a new release changes the llama.cpp build. A maintainer moves them by hand after a benchmark. diff --git a/docs/design/web-console.md b/docs/design/web-console.md index 415d00f..7344640 100644 --- a/docs/design/web-console.md +++ b/docs/design/web-console.md @@ -41,7 +41,7 @@ The console uses Go `html/template` pages on the server with htmx for partial up The pages use one stylesheet with a light and a dark theme from `prefers-color-scheme`. The layout works at phone width. Live views, such as download progress and the router's last decisions, refresh every 5 seconds with `hx-trigger="every 5s"`, as the TUI does. -The model settings form of the console and the TUI's form share `internal/modelform`, so both check a value the same way. +The model settings form of the console and the TUI's form share `internal/modelform`, so both check a value the same way. For an Edge0 model, both forms show only the layout, the context size and the hidden flag, because the other settings are llama-server flags. ## Components diff --git a/docs/development.md b/docs/development.md index 47eb7ea..177ec12 100644 --- a/docs/development.md +++ b/docs/development.md @@ -322,7 +322,7 @@ Renovate (`.github/renovate.json5`) opens a pull request for each new version of - Patch, minor and digest updates merge themselves when every check passes. A major update waits for a maintainer. - The Quadlet `Image=` lines stay digest-only, because the offline bundle reads them as one reference (`scripts/bundle/make.sh`, `internal/bundle`). Renovate updates the version comment and the digest together. - Renovate leaves the ROCm and Ubuntu major and minor releases alone, because a new release changes the llama.cpp build. A maintainer moves them by hand after a benchmark. -- llama.cpp (`LLAMA_CPP_REF`) and llama-swap (`LLAMA_SWAP_VERSION`) move by hand. A llama.cpp release can rename the speculative decoding flags that Sibyl writes. +- llama.cpp (`LLAMA_CPP_REF`) and llama-swap (`LLAMA_SWAP_VERSION`) move by hand, and so does the Edge0 engine (`EDGE0_REF` in `Containerfile.hip`). A llama.cpp release can rename the speculative decoding flags that Sibyl writes. - Renovate's built-in `quadlet` manager is off. It would move the digest-only pins to the newest image of any version, and it reads the `@IMAGE_TAG@` placeholders as digests. ## Release diff --git a/docs/reference/system-api.md b/docs/reference/system-api.md index 10314d2..4e5470c 100644 --- a/docs/reference/system-api.md +++ b/docs/reference/system-api.md @@ -137,15 +137,15 @@ A value that does not fit the variant or the detected GPUs returns `422`. |---|---|---|---| | `GET /models` | None | A list of installed models | Every model in the models directory and in the model pools, with its settings. | | `PUT /models/{id}` | The model settings | The apply result | Replaces the settings of one model and applies them. | -| `DELETE /models/{id}` | None | The apply result | Deletes the model file and all of its shards, removes its settings, and applies. | -| `POST /models/{id}/benchmark` | None | `202`, `{"status": "queued"}` | Asks the host agent to run `llama-bench` on the model with each backend image. | +| `DELETE /models/{id}` | None | The apply result | Deletes the model file and all of its shards, or the folder of an Edge0 model, removes its settings, and applies. | +| `POST /models/{id}/benchmark` | None | `202`, `{"status": "queued"}` | Asks the host agent to run `llama-bench` on the model with each backend image. `422` for an Edge0 model, which llama.cpp cannot run. | | `GET /models/{id}/benchmark` | None | The benchmark state | The progress or the result of the last benchmark. `404` if no benchmark ran. | | `POST /models/{id}/optimize` | `{"profile": "single"}` or `"throughput"`, and an optional `"advisor": {"provider": "...", "model": "..."}` | `202`, the optimizer state | Starts the optimizer for a chat model. It unloads the models while it runs. The advisor must be a provider of the router. `409` while another run goes. | | `GET /models/{id}/optimize` | None | The optimizer state | The stage, every measured result, and the recommendation when the run is done. `404` if no run took place. | | `DELETE /models/{id}/optimize` | None | `202` | Cancels the run. The tool that runs on the host finishes first. | | `POST /models/{id}/optimize/apply` | `{"backend": true}` or `false` | The apply result | Writes the recommended settings. With `backend` true, it also switches the backend of every model when the run found a faster one. `409` if the settings did not load at the model's context. | | `GET /running` | None | The running models | The models that llama-swap runs, the loads that wait, and the last failed load of each model. `502` when llama-swap does not answer. | -| `POST /models/{id}/load` | None | `202`, `{"status": "loading"}` | Starts the model in the background, through llama-swap. llama-swap first stops the models that cannot run with it. `409` while the model is loading, `422` for a drafter. | +| `POST /models/{id}/load` | None | `202`, `{"status": "loading"}` | Starts the model in the background, through llama-swap. llama-swap first stops the models that cannot run with it. `409` while the model is loading, `422` for a drafter, and `422` for an Edge0 model when the host has less free system RAM than the model keeps. | | `POST /models/{id}/unload` | None | `{"status": "unloaded"}` | Stops the model. `422` for a drafter, which stops with its model. | | `GET /models/{id}/drafters?refresh=1` | None | The drafter search | Searches Hugging Face for DFlash and EAGLE-3 drafters of the model. A search is cached for 6 hours. `refresh=1` skips the cache. `422` for a drafter, `502` when Hugging Face fails. | | `POST /models/{id}/drafter-test` | The `speculative` settings to try | `202`, `{"status": "running"}` | Measures the model without and with the drafter, then restores the model's `speculative` settings. It unloads the running models. `409` while another test runs. | @@ -156,6 +156,7 @@ Each model in `GET /models` has these fields: | Field | Content | |---|---| | `id`, `path`, `pool`, `size_bytes`, `shards` | The model, its file, its pool (empty for the models directory), its size and its number of shard files | +| `engine`, `context`, `ram_bytes` | `engine` is `edge0` for an Edge0 model folder, and missing for a GGUF model. `path` is then the folder, `size_bytes` is the size of all its files, `context` is the context length that the folder declares, and `ram_bytes` is the system RAM that the model keeps while it runs on one GPU. A model that spans GPUs keeps less. An Edge0 model has no `meta` and no `fit`. | | `settings` | The settings that the owner chose | | `fit` | The context size and KV cache types that the model runs with, where Sibyl chooses them | | `meta` | What the model file says about its context, and `base_repos`, the Hugging Face repositories in its `general.base_model.*` keys. Missing if the file header cannot be read. | @@ -203,7 +204,7 @@ The benchmark leaves out HIP when a selected GPU has a ROCm target that the HIP | Method and path | Body | Response | What it does | |---|---|---|---| | `GET /hf/search?q=&limit=` | None | A list of repositories | Searches Hugging Face for GGUF repositories, most downloaded first. `q` must be 1 to 200 characters. `limit` is 1 to 100, and the default is 30. | -| `GET /hf/repos/{owner}/{name}?revision=` | None | `repo`, `revision`, `vram_bytes`, `ram_bytes`, `variants` | The complete GGUF variants of a repository at one commit, each with its size and `fit`. Without `revision`, the System API uses the current commit. A given `revision` must be a 40-character commit SHA. | +| `GET /hf/repos/{owner}/{name}?revision=` | None | `repo`, `revision`, `vram_bytes`, `ram_bytes`, `variants` | The complete GGUF variants of a repository at one commit, each with its size and `fit`. An Edge0 repository has one variant, the whole folder, with `engine` set to `edge0` and `ram_bytes`. Its `fit` is `split` when the host's RAM holds the model, and `no` when it does not. Without `revision`, the System API uses the current commit. A given `revision` must be a 40-character commit SHA. | | `GET /downloads` | None | A list of download jobs | Every download job. | | `POST /downloads` | `{"repo", "variant", "revision", "pool"}` | `202` and the new job | Queues a download. `revision` and `pool` are optional. Without `pool`, the model goes to the models directory. | | `GET /downloads/{id}` | None | One job | The progress of one job. `404` if it does not exist. | diff --git a/internal/api/api.go b/internal/api/api.go index 932c981..2e6bf98 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -99,6 +99,9 @@ type cachedMeta struct { // modelMeta reads the GGUF metadata of a model's first file, or returns nil // when the header cannot be read. func (s *Server) modelMeta(m models.Model) *models.Meta { + if m.Engine != "" { + return nil // not a GGUF model: its path is a folder + } path := filepath.Join(m.Root(s.Paths.Models, s.Paths.Pools), m.Path) fi, err := os.Stat(path) if err != nil { @@ -378,6 +381,11 @@ func (s *Server) setModel(id string, set models.Settings) (int, error) { if err != nil { return http.StatusNotFound, err } + // The Edge0 engine has no embedding mode, and the generator would + // leave the model out. + if m.Engine == models.EngineEdge0 && set.Embedding { + return http.StatusUnprocessableEntity, errors.New("embedding: an Edge0 model cannot be an embedding model") + } // llama.cpp refuses a MoE cache on a dense model, and the model would // not load. meta := s.modelMeta(m) @@ -688,6 +696,9 @@ func (s *Server) findModel(id string) (models.Model, error) { // checked to be inside the models directory first. func removeModelFiles(dir string, m models.Model) error { first := filepath.Join(dir, m.Path) + if m.Engine == models.EngineEdge0 { + return removeModelFolder(dir, first) + } paths := []string{first} if m.Shards > 1 { // -00001-of-0000N.gguf -> -*-of-0000N.gguf @@ -722,6 +733,23 @@ func removeModelFiles(dir string, m models.Model) error { return nil } +// removeModelFolder deletes a model that is a whole folder. The folder must +// be inside the models directory, and it must not be that directory. +func removeModelFolder(dir, folder string) error { + root, err := filepath.Abs(dir) + if err != nil { + return err + } + abs, err := filepath.Abs(folder) + if err != nil { + return err + } + if !strings.HasPrefix(abs, root+string(filepath.Separator)) { + return fmt.Errorf("refusing to delete %s: outside %s", abs, root) + } + return os.RemoveAll(abs) +} + const maxBody = 1 << 20 func readJSON(r *http.Request, v any) error { diff --git a/internal/api/api_test.go b/internal/api/api_test.go index a011675..a8636ea 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -480,3 +480,53 @@ func (f *fixture) doRaw(t *testing.T, method, path string) (int, []byte) { } return resp.StatusCode, raw } + +// An Edge0 model folder is one model. It gets the launcher command, the +// llama.cpp tools refuse it, and a delete removes the whole folder. +func TestEdge0Model(t *testing.T) { + f := newFixture(t) + folder := filepath.Join(f.paths.Models, "Edge0-145B-q3k") + write(t, filepath.Join(folder, "model.json"), `{"family":"Edge0","context":262144}`) + write(t, filepath.Join(folder, "experts.bin"), "x") + write(t, filepath.Join(folder, "model-skeleton.gguf"), "x") + write(t, filepath.Join(folder, "mtp", "experts.bin"), "x") + const id = "edge0-145b-q3k" + + f.do(t, "PUT", "/hardware", `{"backend":"hip","gpus":[1,0],"layout":"span"}`, true) + if code, b := f.do(t, "PUT", "/models/"+id, `{"layout":"gpu:1","ctx_size":65536}`, true); code != http.StatusOK { + t.Fatalf("PUT settings = %d %v", code, b) + } + cfg, _ := os.ReadFile(filepath.Join(f.paths.Config, "llama-swap", "config.yaml")) + want := "/opt/edge0/launch.py --model-dir /models/Edge0-145B-q3k --alias " + id + " --gpus 1 --context 65536 --port ${PORT}" + if !strings.Contains(string(cfg), want) { + t.Errorf("config lacks %q:\n%s", want, cfg) + } + // The folder's GGUF file is a part of the model, not a model. + if strings.Contains(string(cfg), "model-skeleton") { + t.Errorf("the folder's GGUF file is listed as a model:\n%s", cfg) + } + + code, raw := f.doRaw(t, "GET", "/models") + if code != http.StatusOK || !strings.Contains(string(raw), `"engine":"edge0"`) || !strings.Contains(string(raw), `"context":262144`) { + t.Errorf("GET /models = %d %s", code, raw) + } + for _, c := range []struct{ method, path, body string }{ + {"PUT", "/models/" + id, `{"embedding":true}`}, + {"POST", "/models/" + id + "/benchmark", ""}, + {"POST", "/models/" + id + "/optimize", `{"profile":"single"}`}, + } { + if code, b := f.do(t, c.method, c.path, c.body, true); code != http.StatusUnprocessableEntity { + t.Errorf("%s %s = %d %v, want 422", c.method, c.path, code, b) + } + } + + if code, b := f.do(t, "DELETE", "/models/"+id, "", true); code != http.StatusOK { + t.Fatalf("DELETE = %d %v", code, b) + } + if _, err := os.Stat(folder); !os.IsNotExist(err) { + t.Errorf("the folder is still there: %v", err) + } + if _, err := os.Stat(filepath.Join(f.paths.Models, "chat-14b.gguf")); err != nil { + t.Errorf("another model went with it: %v", err) + } +} diff --git a/internal/api/bench.go b/internal/api/bench.go index f2ba14f..ce6c43f 100644 --- a/internal/api/bench.go +++ b/internal/api/bench.go @@ -59,6 +59,10 @@ func (s *Server) runnableBackends(hw hardware.Config) []string { return out } +// errLlamaOnly is the answer to a benchmark or an optimizer run for a model +// that llama.cpp does not run. Both tools measure llama.cpp. +var errLlamaOnly = errString("the benchmark and the optimizer run llama.cpp, and another engine runs this model") + func (s *Server) benchDir() string { return filepath.Join(s.Host.Dir, "bench") } func (s *Server) startBenchmark(w http.ResponseWriter, r *http.Request) { @@ -68,6 +72,10 @@ func (s *Server) startBenchmark(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusNotFound, err) return } + if m.Engine != "" { + writeError(w, http.StatusUnprocessableEntity, errLlamaOnly) + return + } hw, err := s.hardware() if err != nil { writeError(w, http.StatusInternalServerError, err) diff --git a/internal/api/models_hf.go b/internal/api/models_hf.go index cdb9a87..13a9d59 100644 --- a/internal/api/models_hf.go +++ b/internal/api/models_hf.go @@ -10,6 +10,7 @@ import ( "github.com/diffsec/sibyl/internal/gpu" "github.com/diffsec/sibyl/internal/hardware" "github.com/diffsec/sibyl/internal/hf" + "github.com/diffsec/sibyl/internal/models" ) // Fit says how a model of a given size can run on this host. @@ -77,16 +78,26 @@ func (s *Server) hfRepo(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, errString("revision must be a 40-character commit SHA")) return } - files, err := s.HF.Files(r.Context(), repo, rev) + variants, err := s.HF.RepoVariants(r.Context(), repo, rev) if err != nil { writeError(w, http.StatusBadGateway, err) return } vram, ram, known := s.memory() out := repoView{Repo: repo, Revision: rev, VRAM: vram, RAM: ram} - for _, v := range hf.Variants(files) { + for _, v := range variants { f := FitUnknown - if known { + switch { + case v.Engine == models.EngineEdge0: + // The engine keeps the expert file in system RAM and fills + // the VRAM it finds, so RAM decides the fit. + if ram > 0 { + f = FitNo + if v.RAM <= ram { + f = FitSplit + } + } + case known: f = fit(v.Size, vram, ram) } out.Variants = append(out.Variants, variantView{Variant: v, Fit: f}) @@ -122,17 +133,25 @@ func (s *Server) memory() (vram, ram int64, known bool) { } } } - if f, err := os.Open(s.Paths.MemInfo); err == nil { - defer func() { _ = f.Close() }() - sc := bufio.NewScanner(f) - for sc.Scan() { - if fields := strings.Fields(sc.Text()); len(fields) >= 2 && fields[0] == "MemTotal:" { - kb, _ := strconv.ParseInt(fields[1], 10, 64) - ram = kb << 10 - } + return vram, s.memInfo("MemTotal:"), known +} + +// memInfo returns one value of /proc/meminfo in bytes, or 0 when the file +// or the field cannot be read. field includes the colon, as in "MemTotal:". +func (s *Server) memInfo(field string) int64 { + f, err := os.Open(s.Paths.MemInfo) + if err != nil { + return 0 + } + defer func() { _ = f.Close() }() + sc := bufio.NewScanner(f) + for sc.Scan() { + if fields := strings.Fields(sc.Text()); len(fields) >= 2 && fields[0] == field { + kb, _ := strconv.ParseInt(fields[1], 10, 64) + return kb << 10 } } - return vram, ram, known + return 0 } type downloadRequest struct { diff --git a/internal/api/optimize.go b/internal/api/optimize.go index d6dd519..8dfbf1c 100644 --- a/internal/api/optimize.go +++ b/internal/api/optimize.go @@ -187,6 +187,9 @@ func (s *Server) optimizeInput(id string, profile optimize.Profile, adv *Optimiz if err != nil { return optimize.Input{}, nil, fmt.Errorf("%w: %v", os.ErrNotExist, err) } + if m.Engine != "" { + return optimize.Input{}, nil, errLlamaOnly + } settings, err := models.LoadSettings(s.Paths.settingsFile()) if err != nil { return optimize.Input{}, nil, err diff --git a/internal/api/router.go b/internal/api/router.go index 15b715b..0a0b99b 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -47,6 +47,9 @@ func (s *Server) writeRouterModels(installed []models.Model, settings map[string st := settings[m.ID] meta := s.modelMeta(m) lm := router.LocalModel{ID: m.ID, Context: st.CtxSize, Embedding: st.Embedding} + if m.Engine == models.EngineEdge0 { + lm.Context = m.Edge0Context(st.CtxSize) + } // With several slots, each request is sure of its share only. if st.Parallel > 1 { lm.Context = st.CtxSize / st.Parallel diff --git a/internal/api/running.go b/internal/api/running.go index 80589a2..3edc501 100644 --- a/internal/api/running.go +++ b/internal/api/running.go @@ -13,6 +13,8 @@ import ( "sync" "time" + "github.com/diffsec/sibyl/internal/gpu" + "github.com/diffsec/sibyl/internal/hardware" "github.com/diffsec/sibyl/internal/models" ) @@ -157,6 +159,10 @@ func (s *Server) loadModel(w http.ResponseWriter, r *http.Request) { if !s.servedModel(w, id) { return } + if err := s.ramShort(r.Context(), id); err != nil { + writeError(w, http.StatusUnprocessableEntity, err) + return + } if !s.loads.start(id) { writeError(w, http.StatusConflict, errString("the model is already loading")) return @@ -173,6 +179,57 @@ func (s *Server) loadModel(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusAccepted, map[string]string{"status": "loading"}) } +// ramShort returns an error when the host does not have the free system +// RAM that model id keeps while it runs. Only an Edge0 model has such a +// need. A model that llama-swap already runs holds its RAM, so it passes. +// The launcher in the image makes the same check, for a load that a chat +// request starts. +func (s *Server) ramShort(ctx context.Context, id string) error { + m, err := s.findModel(id) + if err != nil || m.RAM == 0 { + return nil + } + need := m.Edge0RAMOn(s.modelGPUs(id)) + avail := s.memInfo("MemAvailable:") + if avail == 0 || avail >= need { + return nil // enough, or the host does not report it + } + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + if body, err := s.llamaSwapDo(ctx, http.MethodGet, "/running"); err == nil { + var raw struct { + Running []struct { + Model string `json:"model"` + } `json:"running"` + } + if json.Unmarshal(body, &raw) == nil { + for _, r := range raw.Running { + if r.Model == id { + return nil + } + } + } + } + return fmt.Errorf("the model keeps %.1f GiB of system RAM, and %.1f GiB is available", + float64(need)/(1<<30), float64(avail)/(1<<30)) +} + +// modelGPUs returns the number of GPUs that model id runs on: every +// selected GPU for a model that spans them, else 1. It answers 1 when the +// hardware or the settings cannot be read, which is the larger RAM need. +func (s *Server) modelGPUs(id string) int { + hw, err := s.hardware() + if err != nil { + return 1 + } + settings, err := models.LoadSettings(s.Paths.settingsFile()) + if err != nil { + return 1 + } + gpus, _ := gpu.Detect(s.Paths.SysfsRoot) + return max(1, len(hardware.ModelGPUs(settings[id].Layout, false, hw, gpus))) +} + func (s *Server) unloadModel(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") if !s.servedModel(w, id) { diff --git a/internal/api/running_test.go b/internal/api/running_test.go index 570d3db..efaa019 100644 --- a/internal/api/running_test.go +++ b/internal/api/running_test.go @@ -2,9 +2,11 @@ package api import ( "encoding/json" + "fmt" "net/http" "net/http/httptest" "path/filepath" + "strings" "sync" "testing" "time" @@ -91,3 +93,83 @@ func TestRunningLoadUnload(t *testing.T) { } } } + +// An Edge0 model keeps its expert file in system RAM. A load is refused +// when the host does not have that RAM free, unless the model already runs. +func TestLoadEdge0NeedsRAM(t *testing.T) { + f := newFixture(t) + var mu sync.Mutex + var running []map[string]string + loads := 0 + swap := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + switch r.URL.Path { + case "/running": + _ = json.NewEncoder(w).Encode(map[string]any{"running": running}) + case "/upstream/edge0/health": + loads++ + } + })) + t.Cleanup(swap.Close) + f.server.LlamaSwapURL = swap.URL + f.server.Paths.MemInfo = filepath.Join(t.TempDir(), "meminfo") + folder := filepath.Join(f.paths.Models, "edge0") + write(t, filepath.Join(folder, "model.json"), `{"family":"Edge0","context":262144}`) + write(t, filepath.Join(folder, "experts.bin"), strings.Repeat("x", 1<<20)) // 1 MiB, so 6 GiB + 1 MiB of RAM + meminfo := func(availGiB int) { + write(t, f.server.Paths.MemInfo, fmt.Sprintf("MemTotal: %d kB\nMemAvailable: %d kB\n", 64<<20, availGiB<<20)) + } + wait := func() { + for range 200 { + code, body := f.doRaw(t, "GET", "/running") + var r Running + _ = json.Unmarshal(body, &r) + if code == http.StatusOK && len(r.Loading) == 0 { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("the load did not finish") + } + + meminfo(4) + code, b := f.do(t, "POST", "/models/edge0/load", "", true) + if code != http.StatusUnprocessableEntity || !strings.Contains(fmt.Sprint(b["error"]), "4.0 GiB is available") { + t.Fatalf("load with 4 GiB free = %d %v, want 422 that names the RAM", code, b) + } + // A GGUF model has no RAM need. + if code, b := f.do(t, "POST", "/models/chat-14b/load", "", true); code != http.StatusAccepted { + t.Errorf("GGUF load with 4 GiB free = %d %v", code, b) + } + wait() + // The running model holds its RAM already. + mu.Lock() + running = []map[string]string{{"model": "edge0", "state": "ready"}} + mu.Unlock() + if code, b := f.do(t, "POST", "/models/edge0/load", "", true); code != http.StatusAccepted { + t.Errorf("load of the running model = %d %v", code, b) + } + wait() + mu.Lock() + running = nil + mu.Unlock() + meminfo(8) + if code, b := f.do(t, "POST", "/models/edge0/load", "", true); code != http.StatusAccepted { + t.Errorf("load with 8 GiB free = %d %v", code, b) + } + wait() + // A model that spans two GPUs keeps less RAM: the launcher starts the + // engine in its low-RAM mode. 5 GiB is under the 6 GiB that one GPU + // needs, and it is under the span's need too. + f.do(t, "PUT", "/hardware", `{"backend":"hip","gpus":[1,0],"layout":"span"}`, true) + meminfo(5) + if code, b := f.do(t, "POST", "/models/edge0/load", "", true); code != http.StatusUnprocessableEntity { + t.Errorf("span load with 5 GiB free = %d %v, want 422", code, b) + } + mu.Lock() + defer mu.Unlock() + if loads != 2 { + t.Errorf("llama-swap got %d loads, want 2", loads) + } +} diff --git a/internal/console/models.go b/internal/console/models.go index f831077..c7b50e4 100644 --- a/internal/console/models.go +++ b/internal/console/models.go @@ -175,6 +175,9 @@ func (c *Console) modelPage(w http.ResponseWriter, r *http.Request) { } } head := fmt.Sprintf("%s · %d file(s)", gib(m.Size), m.Shards) + if m.Engine == models.EngineEdge0 { + head = fmt.Sprintf("%s · Edge0 model folder · native context %d tokens", gib(m.Size), m.Context) + } moe := false if m.Meta != nil { head += fmt.Sprintf(" · %s · native context %d tokens", m.Meta.Architecture, m.Meta.ContextLength) @@ -190,6 +193,7 @@ func (c *Console) modelPage(w http.ResponseWriter, r *http.Request) { "Model": m, "Head": head, "Fit": fitText(m.Fit), "Form": modelform.FromSettings(m.Settings), "Layouts": layouts, "Drafts": drafts, "SimpleDrafts": simple, "CacheTypes": models.CacheTypes, "Presets": models.Presets, "SpecTypes": spec, "MoE": moe, "Reasoning": reasoningControl(m), "Advisors": advisors, + "Edge0Context": m.Edge0Context(0), } { data[k] = v } @@ -332,10 +336,16 @@ func (c *Console) repoPage(w http.ResponseWriter, r *http.Request) { type row struct { Name, Size, Fit string Shards int + // RAM is set for an Edge0 model folder: the system RAM it keeps. + RAM string } var rows []row for _, v := range repo.Variants { - rows = append(rows, row{v.Name, gib(v.Size), v.Fit, v.Shards}) + rw := row{Name: v.Name, Size: gib(v.Size), Fit: v.Fit, Shards: v.Shards} + if v.Engine == models.EngineEdge0 { + rw.RAM = gib(v.RAM) + } + rows = append(rows, rw) } data["ID"], data["Repo"], data["Variants"], data["Pools"] = id, repo, rows, pools.Pools data["VRAM"], data["RAM"] = gib(repo.VRAM), gib(repo.RAM) diff --git a/internal/console/templates/model.html b/internal/console/templates/model.html index bb5b302..10f2b13 100644 --- a/internal/console/templates/model.html +++ b/internal/console/templates/model.html @@ -2,8 +2,24 @@ {{define "content"}}

Back to the models

{{with .FlashError}}{{end}} -{{template "optimizer" .}} {{$f := .Form}} +{{if eq .Model.Engine "edge0"}} +
+ +
+

{{.Model.ID}}

+

{{.Head}}

+

The Edge0 engine runs this model, not llama.cpp. A model that spans the GPUs runs as a layer split, and no other chat model runs beside it. A model pinned to one GPU leaves the other GPUs to other models.

+
+ + +
+ +
+ +
+{{else}} +{{template "optimizer" .}}
@@ -96,6 +112,7 @@

Advanced

{{end}} +{{end}} {{define "optimizer"}}
diff --git a/internal/console/templates/repo.html b/internal/console/templates/repo.html index bde208c..4946021 100644 --- a/internal/console/templates/repo.html +++ b/internal/console/templates/repo.html @@ -10,7 +10,7 @@

{{.ID}}

{{range .Variants}} {{.Name}}{{.Size}}{{if gt .Shards 1}} in {{.Shards}} files{{end}} - {{if eq .Fit "gpu"}}fits in VRAM{{else if eq .Fit "split"}}needs system RAM too{{else if eq .Fit "unknown"}}unknown{{else}}too large{{end}} + {{if .RAM}}Edge0 model folder. {{if eq .Fit "no"}}It keeps {{.RAM}} of system RAM, and this host has {{$.RAM}}{{else}}It keeps {{.RAM}} of system RAM{{end}}{{else if eq .Fit "gpu"}}fits in VRAM{{else if eq .Fit "split"}}needs system RAM too{{else if eq .Fit "unknown"}}unknown{{else}}too large{{end}}
{{if $.Pools}}{{end}} diff --git a/internal/download/download.go b/internal/download/download.go index 75bea01..9ba5167 100644 --- a/internal/download/download.go +++ b/internal/download/download.go @@ -8,6 +8,7 @@ package download import ( "context" "crypto/rand" + "crypto/sha1" "crypto/sha256" "encoding/hex" "encoding/json" @@ -181,11 +182,11 @@ func (m *Manager) Add(ctx context.Context, repo, revision, variant, pool string) if !hf.ValidRevision(revision) { return Job{}, fmt.Errorf("revision must be a 40-character commit SHA, got %q", revision) } - files, err := m.HF.Files(ctx, repo, revision) + variants, err := m.HF.RepoVariants(ctx, repo, revision) if err != nil { return Job{}, err } - v, err := hf.FindVariant(hf.Variants(files), variant) + v, err := hf.FindVariant(variants, variant) if err != nil { return Job{}, err } @@ -367,7 +368,7 @@ func (m *Manager) fetchAll(ctx context.Context, j *Job) error { } m.setStatus(j, Verifying, "") for _, f := range j.Files { - if err := verify(filepath.Join(partial, filepath.Base(f.Path)), f.SHA256); err != nil { + if err := verify(filepath.Join(partial, destOf(f)), f); err != nil { return fmt.Errorf("%s: %w", f.Path, err) } } @@ -391,7 +392,7 @@ func (m *Manager) fetchAll(ctx context.Context, j *Job) error { func (m *Manager) checkSpace(j *Job, partial string) error { var have int64 for _, f := range j.Files { - if st, err := os.Stat(filepath.Join(partial, filepath.Base(f.Path))); err == nil { + if st, err := os.Stat(filepath.Join(partial, destOf(f))); err == nil { have += st.Size() } } @@ -410,7 +411,11 @@ func (m *Manager) checkSpace(j *Job, partial string) error { // fetchFile downloads one file, resuming from a partial file and retrying // with backoff. Each attempt fails if no data arrives for StallTimeout. func (m *Manager) fetchFile(ctx context.Context, j *Job, dir string, f hf.File) error { - dest := filepath.Join(dir, filepath.Base(f.Path)) + dest := filepath.Join(dir, destOf(f)) + // A file of a model folder can be in a subfolder. + if err := os.MkdirAll(filepath.Dir(dest), 0o750); err != nil { + return err + } var lastErr error for attempt := 0; attempt < 10; attempt++ { if attempt > 0 { @@ -522,7 +527,7 @@ func (m *Manager) attempt(parent context.Context, j *Job, dest string, f hf.File func (m *Manager) progress(j *Job) { var done int64 for _, f := range j.Files { - if st, err := os.Stat(filepath.Join(m.partialDir(j), filepath.Base(f.Path))); err == nil { + if st, err := os.Stat(filepath.Join(m.partialDir(j), destOf(f))); err == nil { done += st.Size() } } @@ -531,19 +536,47 @@ func (m *Manager) progress(j *Job) { m.mu.Unlock() } -func verify(path, want string) error { - f, err := os.Open(path) +// destOf returns where a file goes inside the model folder: its Dest, or +// its base name. A Dest that is not a plain relative path falls back to the +// base name, so a file never lands outside the folder. +func destOf(f hf.File) string { + if f.Dest != "" && hf.SafePath(f.Dest) { + return filepath.FromSlash(f.Dest) + } + return filepath.Base(f.Path) +} + +// verify checks a downloaded file against its SHA-256, or against its Git +// blob ID when Hugging Face gives no SHA-256 for it. A Git blob ID is the +// SHA-1 of "blob \x00" and the content. +func verify(path string, f hf.File) error { + file, err := os.Open(path) if err != nil { return err } - defer func() { _ = f.Close() }() - h := sha256.New() - if _, err := io.Copy(h, f); err != nil { + defer func() { _ = file.Close() }() + want, name, h := f.SHA256, "SHA-256", sha256.New() + if want == "" { + fi, err := file.Stat() + if err != nil { + return err + } + // SHA-1 is what Git names a blob with. The commit that pins the + // download is itself a SHA-1. + want, name, h = f.GitOID, "Git blob ID", sha1.New() + if _, err := fmt.Fprintf(h, "blob %d\x00", fi.Size()); err != nil { + return err + } + } + if want == "" { + return errors.New("no digest to verify against") + } + if _, err := io.Copy(h, file); err != nil { return err } if got := hex.EncodeToString(h.Sum(nil)); got != want { _ = os.Remove(path) // a corrupt file must be downloaded again - return fmt.Errorf("SHA-256 mismatch: got %s, want %s", got, want) + return fmt.Errorf("%s mismatch: got %s, want %s", name, got, want) } return nil } diff --git a/internal/download/download_test.go b/internal/download/download_test.go index 8c7adfd..a468cb8 100644 --- a/internal/download/download_test.go +++ b/internal/download/download_test.go @@ -3,6 +3,7 @@ package download import ( "bytes" "context" + "crypto/sha1" "crypto/sha256" "encoding/hex" "encoding/json" @@ -27,6 +28,7 @@ const rev = "4c577be098c274a1f579e072178db01dd3a8e449" type fakeHub struct { files map[string][]byte shaFor map[string]string // override the advertised digest + plain map[string]bool // path -> true: not in LFS, so the tree has its Git blob ID only cutOnce sync.Map // path -> true: first GET stops halfway gets atomic.Int32 } @@ -36,6 +38,14 @@ func digest(b []byte) string { return hex.EncodeToString(s[:]) } +// gitBlob is the Git blob ID of b. +func gitBlob(b []byte) string { + h := sha1.New() + _, _ = fmt.Fprintf(h, "blob %d\x00", len(b)) + _, _ = h.Write(b) + return hex.EncodeToString(h.Sum(nil)) +} + func (h *fakeHub) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch { case r.URL.Path == "/api/models/o/r": @@ -47,6 +57,14 @@ func (h *fakeHub) ServeHTTP(w http.ResponseWriter, r *http.Request) { if o, ok := h.shaFor[p]; ok { sha = o } + if h.plain[p] { + oid := gitBlob(b) + if o, ok := h.shaFor[p]; ok { + oid = o + } + tree = append(tree, map[string]any{"type": "file", "path": p, "size": len(b), "oid": oid}) + continue + } tree = append(tree, map[string]any{"type": "file", "path": p, "size": len(b), "lfs": map[string]any{"oid": sha}}) } _ = json.NewEncoder(w).Encode(tree) @@ -231,3 +249,68 @@ func TestDirName(t *testing.T) { } } } + +// An Edge0 repository downloads as one folder: the files keep their +// subfolders, a file outside LFS is checked against its Git blob ID, and the +// model card's assets stay out. +func TestDownloadEdge0Folder(t *testing.T) { + hub := &fakeHub{ + files: map[string][]byte{ + "model.json": []byte(`{"family":"Edge0","context":262144}`), + "experts.bin": bytes.Repeat([]byte("e"), 4096), + "model-skeleton.gguf": bytes.Repeat([]byte("g"), 2048), + "mtp/experts.bin": bytes.Repeat([]byte("m"), 1024), + "tokenizer/vocab.json": []byte(`{"a":1}`), + "assets/hero.png": []byte("png"), + ".gitattributes": []byte("x"), + }, + plain: map[string]bool{"model.json": true, "tokenizer/vocab.json": true, ".gitattributes": true}, + } + m, done := newManager(t, hub) + // The GGUF file is a part of the folder, not a variant of its own. + if _, err := m.Add(context.Background(), "o/r", "", "model-skeleton", ""); err == nil { + t.Error("the folder's GGUF file was offered as a variant") + } + j, err := m.Add(context.Background(), "o/r", "", "r", "") + if err != nil { + t.Fatal(err) + } + if j.Dir != "r" || len(j.Files) != 5 { + t.Fatalf("job = %+v", j) + } + waitFor(t, m, j.ID, Done) + <-done + for p, want := range hub.files { + got, err := os.ReadFile(filepath.Join(m.ModelsDir, "r", filepath.FromSlash(p))) + skipped := strings.HasPrefix(p, "assets/") || strings.HasPrefix(p, ".") + switch { + case skipped && err == nil: + t.Errorf("%s was downloaded", p) + case !skipped && (err != nil || !bytes.Equal(got, want)): + t.Errorf("%s: err = %v, %d bytes, want %d", p, err, len(got), len(want)) + } + } +} + +func TestDownloadEdge0GitBlobMismatch(t *testing.T) { + hub := &fakeHub{ + files: map[string][]byte{ + "model.json": []byte(`{"family":"Edge0"}`), + "experts.bin": bytes.Repeat([]byte("e"), 512), + }, + plain: map[string]bool{"model.json": true}, + shaFor: map[string]string{"model.json": strings.Repeat("ab", 20)}, + } + m, _ := newManager(t, hub) + j, err := m.Add(context.Background(), "o/r", "", "r", "") + if err != nil { + t.Fatal(err) + } + got := waitFor(t, m, j.ID, Failed) + if !strings.Contains(got.Error, "Git blob ID mismatch") { + t.Errorf("error = %q", got.Error) + } + if _, err := os.Stat(filepath.Join(m.ModelsDir, "r")); err == nil { + t.Error("a folder with a bad file was installed") + } +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index cee2e1b..68f14f4 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -17,6 +17,8 @@ import ( "strings" "sync" "time" + + "github.com/diffsec/sibyl/internal/models" ) // DefaultBase is the Hub address. @@ -26,6 +28,7 @@ var ( repoRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,95}/[A-Za-z0-9][A-Za-z0-9._-]{0,95}$`) revisionRe = regexp.MustCompile(`^[0-9a-f]{40}$`) sha256Re = regexp.MustCompile(`^[0-9a-f]{64}$`) + gitOIDRe = regexp.MustCompile(`^[0-9a-f]{40}$`) // a Git blob ID: SHA-1 splitRe = regexp.MustCompile(`^(.*)-([0-9]{5})-of-([0-9]{5})\.gguf$`) ) @@ -122,14 +125,26 @@ type File struct { Path string `json:"path"` Size int64 `json:"size"` SHA256 string `json:"sha256"` + // GitOID is the Git blob ID of a file that is not in LFS. Hugging Face + // gives a SHA-256 for LFS files only, so a small file is checked + // against this SHA-1 of its Git blob. + GitOID string `json:"git_oid,omitempty"` + // Dest is the file's path inside the model folder. An empty Dest means + // the base name of Path, which is where a GGUF file goes. + Dest string `json:"dest,omitempty"` } -// Variant is one downloadable GGUF model: a single file or a full split set. +// Variant is one downloadable model: a single GGUF file, a full split set, +// or a whole Edge0 model folder. type Variant struct { Name string `json:"name"` Size int64 `json:"size_bytes"` Shards int `json:"shards"` Files []File `json:"files"` + // Engine is "edge0" for an Edge0 model folder, and empty for GGUF. + Engine string `json:"engine,omitempty"` + // RAM is the system memory that an Edge0 model keeps while it runs. + RAM int64 `json:"ram_bytes,omitempty"` } func (c *Client) get(ctx context.Context, rawURL string, v any) error { @@ -210,6 +225,80 @@ func (c *Client) Files(ctx context.Context, repo, rev string) ([]File, error) { return out, nil } +// Edge0Variant returns the whole-folder variant of an Edge0 repository, or +// false for any other repository. An Edge0 repository holds a strata pack at +// its root: model.json and the expert file. The variant leaves out the +// repository's assets folder, which holds pictures for the model card. +func (c *Client) Edge0Variant(ctx context.Context, repo, rev string) (Variant, bool, error) { + if !ValidRepo(repo) || !ValidRevision(rev) { + return Variant{}, false, fmt.Errorf("invalid repository %q or revision %q", repo, rev) + } + var tree []struct { + Type string `json:"type"` + Path string `json:"path"` + Size int64 `json:"size"` + OID string `json:"oid"` + LFS *struct { + OID string `json:"oid"` + } `json:"lfs"` + } + if err := c.get(ctx, c.Base+"/api/models/"+repo+"/tree/"+rev+"?recursive=true", &tree); err != nil { + return Variant{}, false, err + } + v := Variant{Name: path.Base(repo), Engine: models.EngineEdge0} + var manifest, experts bool + for _, e := range tree { + if e.Type != "file" || strings.HasPrefix(e.Path, "assets/") { + continue + } + f := File{Path: e.Path, Size: e.Size, Dest: e.Path} + switch { + case !SafePath(e.Path): + continue // a hidden file such as .gitattributes, or an unsafe name + case e.LFS != nil && sha256Re.MatchString(e.LFS.OID): + f.SHA256 = e.LFS.OID + case e.LFS == nil && gitOIDRe.MatchString(e.OID): + f.GitOID = e.OID + default: + continue // no digest to verify against + } + switch e.Path { + case "model.json": + manifest = true + case "experts.bin": + experts = true + v.RAM = models.Edge0RAM(e.Size) + } + v.Files = append(v.Files, f) + v.Size += e.Size + v.Shards++ + } + if !manifest || !experts { + return Variant{}, false, nil + } + sort.Slice(v.Files, func(i, j int) bool { return v.Files[i].Path < v.Files[j].Path }) + return v, true, nil +} + +// RepoVariants lists what a repository offers for download: its GGUF +// variants, and the whole folder when the repository is an Edge0 model. The +// GGUF files of an Edge0 repository are parts of the folder, so they are not +// offered alone. +func (c *Client) RepoVariants(ctx context.Context, repo, rev string) ([]Variant, error) { + folder, ok, err := c.Edge0Variant(ctx, repo, rev) + if err != nil { + return nil, err + } + if ok { + return []Variant{folder}, nil + } + files, err := c.Files(ctx, repo, rev) + if err != nil { + return nil, err + } + return Variants(files), nil +} + // SafePath reports whether a repository path is a plain relative path. func SafePath(p string) bool { if p == "" || strings.HasPrefix(p, "/") || strings.Contains(p, "\\") || strings.ContainsAny(p, "\x00\n\r") { diff --git a/internal/hf/hf_test.go b/internal/hf/hf_test.go index 621715f..4940d6c 100644 --- a/internal/hf/hf_test.go +++ b/internal/hf/hf_test.go @@ -1,6 +1,14 @@ package hf -import "testing" +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/diffsec/sibyl/internal/models" +) func TestVariants(t *testing.T) { files := []File{ @@ -48,3 +56,62 @@ func TestFileURLEscapes(t *testing.T) { t.Errorf("FileURL = %s", got) } } + +const testRev = "4c577be098c274a1f579e072178db01dd3a8e449" + +func treeServer(t *testing.T, tree string) *Client { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.URL.Path, "/api/models/o/Edge0-145B-q3k/tree/"+testRev) { + http.NotFound(w, r) + return + } + _, _ = w.Write([]byte(tree)) + })) + t.Cleanup(srv.Close) + return New(srv.URL, "") +} + +// An Edge0 repository is offered as one folder. Its GGUF files are parts +// of the folder, so they are not offered alone. +func TestRepoVariantsEdge0(t *testing.T) { + sha, blob := strings.Repeat("a", 64), strings.Repeat("b", 40) + c := treeServer(t, `[ + {"type":"directory","path":"mtp","size":0,"oid":"`+blob+`"}, + {"type":"file","path":".gitattributes","size":10,"oid":"`+blob+`"}, + {"type":"file","path":"assets/hero.png","size":99,"oid":"`+blob+`","lfs":{"oid":"`+sha+`"}}, + {"type":"file","path":"model.json","size":20,"oid":"`+blob+`"}, + {"type":"file","path":"experts.bin","size":4000,"oid":"`+blob+`","lfs":{"oid":"`+sha+`"}}, + {"type":"file","path":"model-skeleton.gguf","size":600,"oid":"`+blob+`","lfs":{"oid":"`+sha+`"}}, + {"type":"file","path":"mtp/experts.bin","size":70,"oid":"`+blob+`","lfs":{"oid":"`+sha+`"}}, + {"type":"file","path":"start.sh","size":5,"oid":"not-a-digest"} + ]`) + vs, err := c.RepoVariants(context.Background(), "o/Edge0-145B-q3k", testRev) + if err != nil || len(vs) != 1 { + t.Fatalf("variants = %+v, err = %v", vs, err) + } + v := vs[0] + if v.Name != "Edge0-145B-q3k" || v.Engine != models.EngineEdge0 || v.Size != 4690 || v.Shards != 4 || + v.RAM != 4000+models.Edge0RAMOverhead { + t.Errorf("variant = %+v", v) + } + for _, f := range v.Files { + lfs := f.Path != "model.json" + if f.Dest != f.Path || (f.SHA256 != "") != lfs || (f.GitOID != "") == lfs { + t.Errorf("file = %+v", f) + } + } +} + +// A repository with no strata pack at its root keeps its GGUF variants. +func TestRepoVariantsGGUF(t *testing.T) { + sha := strings.Repeat("a", 64) + c := treeServer(t, `[ + {"type":"file","path":"model.json","size":20,"oid":"`+strings.Repeat("b", 40)+`"}, + {"type":"file","path":"Model-Q4_K_M.gguf","size":600,"lfs":{"oid":"`+sha+`"}} + ]`) + vs, err := c.RepoVariants(context.Background(), "o/Edge0-145B-q3k", testRev) + if err != nil || len(vs) != 1 || vs[0].Name != "Model-Q4_K_M" || vs[0].Engine != "" { + t.Fatalf("variants = %+v, err = %v", vs, err) + } +} diff --git a/internal/llamaswap/generate.go b/internal/llamaswap/generate.go index d7b37cd..d50f40b 100644 --- a/internal/llamaswap/generate.go +++ b/internal/llamaswap/generate.go @@ -126,9 +126,18 @@ func Generate(hw hardware.Config, installed []models.Model, settings map[string] } } p := place(hw, s, selected) + cmd := command(m, s, p, selected, devPrefix, draftPath) + if m.Engine == models.EngineEdge0 { + // The engine is built for HIP only, and it has no embedding mode. + if hw.Backend != hardware.BackendHIP || s.Embedding { + issues = append(issues, Issue{ID: m.ID, Reason: "an Edge0 model needs the hip backend, and it cannot be an embedding model", Skipped: true}) + continue + } + cmd = edge0Command(m, s, p, selected) + } cfg.Models[m.ID] = ModelEntry{ Name: m.ID, - Cmd: command(m, s, p, selected, devPrefix, draftPath), + Cmd: cmd, Unlisted: s.Hidden, } switch { @@ -228,6 +237,34 @@ func command(m models.Model, s models.Settings, p placement, selected int, devPr return strings.Join(args, " ") } +// Edge0Launcher starts edge0-strata's server for one model folder. It is in +// the HIP llama-swap image. +const Edge0Launcher = "/opt/venv/bin/python /opt/edge0/launch.py" + +// edge0Command is the launcher command for an Edge0 model folder. The engine +// takes GPU numbers as HIP numbers them inside the container, which is the +// position in the selected GPU list. A spanning model is a layer split across +// every selected GPU. The llama.cpp settings do not apply to this engine. +func edge0Command(m models.Model, s models.Settings, p placement, selected int) string { + gpus := strconv.Itoa(p.gpu) + if p.span && selected > 1 { + parts := make([]string, selected) + for i := range parts { + parts[i] = strconv.Itoa(i) + } + gpus = strings.Join(parts, ",") + } + args := []string{ + Edge0Launcher, + "--model-dir", modelPath(m), + "--alias", m.ID, + "--gpus", gpus, + "--context", strconv.Itoa(m.Edge0Context(s.CtxSize)), + "--port", "${PORT}", + } + return strings.Join(args, " ") +} + // matrix lists which models can run together: // - one chat model per GPU, plus every embedding model; // - or one spanning model, plus every embedding model. diff --git a/internal/llamaswap/generate_test.go b/internal/llamaswap/generate_test.go index 1e62832..64686da 100644 --- a/internal/llamaswap/generate_test.go +++ b/internal/llamaswap/generate_test.go @@ -289,3 +289,52 @@ func TestGenerateReasoningEffort(t *testing.T) { t.Errorf("an embedding model got the reasoning effort: %s", cfg.Models["embed"].Cmd) } } + +func TestGenerateEdge0(t *testing.T) { + edge0 := models.Model{ID: "edge0-145b-q3k", Path: "Edge0-145B-q3k", Engine: models.EngineEdge0, Context: 262144} + chat := models.Model{ID: "chat-14b", Path: "chat-14b-Q8_0.gguf"} + both := []models.Model{chat, edge0} + + // Spanning: a layer split across every selected GPU, in a set of its own. + cfg, issues := Generate(twoGPUs(hardware.LayoutSpan), both, nil) + if len(issues) != 0 { + t.Fatalf("issues = %+v", issues) + } + want := Edge0Launcher + " --model-dir /models/Edge0-145B-q3k --alias edge0-145b-q3k --gpus 0,1 --context 131072 --port ${PORT}" + if cmd := cfg.Models["edge0-145b-q3k"].Cmd; cmd != want { + t.Errorf("span cmd = %q, want %q", cmd, want) + } + mx := cfg.Routing.Router.Settings.Matrix + if len(mx.Sets) != 2 || mx.Sets["pinned"] != "" { + t.Errorf("two spanning models must each run alone: sets = %+v", mx.Sets) + } + + // Pinned to GPU 1: the chat model on GPU 0 can run beside it. + settings := map[string]models.Settings{ + "edge0-145b-q3k": {Layout: "gpu:1", CtxSize: 65536}, + "chat-14b": {Layout: "gpu:0"}, + } + cfg, issues = Generate(twoGPUs(hardware.LayoutSpan), both, settings) + if len(issues) != 0 { + t.Fatalf("issues = %+v", issues) + } + want = Edge0Launcher + " --model-dir /models/Edge0-145B-q3k --alias edge0-145b-q3k --gpus 1 --context 65536 --port ${PORT}" + if cmd := cfg.Models["edge0-145b-q3k"].Cmd; cmd != want { + t.Errorf("pinned cmd = %q, want %q", cmd, want) + } + mx = cfg.Routing.Router.Settings.Matrix + if len(mx.Sets) != 1 || strings.Count(mx.Sets["pinned"], "&") != 1 { + t.Errorf("models on different GPUs must share one set: sets = %+v", mx.Sets) + } + + // The engine is built for HIP only. + hw := twoGPUs(hardware.LayoutSpan) + hw.Backend = hardware.BackendVulkan + cfg, issues = Generate(hw, both, nil) + if _, ok := cfg.Models["edge0-145b-q3k"]; ok || len(issues) != 1 || !issues[0].Skipped { + t.Errorf("want the Edge0 model skipped on vulkan: models = %+v, issues = %+v", cfg.Models, issues) + } + if _, ok := cfg.Models["chat-14b"]; !ok { + t.Error("the GGUF model must stay") + } +} diff --git a/internal/models/models.go b/internal/models/models.go index b3804f5..4e576e5 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -1,7 +1,8 @@ -// Package models lists installed GGUF models and stores per-model settings. +// Package models lists installed models and stores per-model settings. package models import ( + "encoding/json" "errors" "fmt" "io/fs" @@ -19,10 +20,23 @@ import ( "github.com/diffsec/sibyl/internal/hardware" ) -// Model is one installed GGUF model. A split model is listed once, by its first shard. +// EngineEdge0 is the Engine of an Edge0 model folder, which edge0-strata +// runs. A GGUF model has an empty Engine, and llama.cpp runs it. +const EngineEdge0 = "edge0" + +// Model is one installed model: a GGUF model, listed once by its first +// shard, or an Edge0 model folder. type Model struct { ID string `json:"id"` - Path string `json:"path"` // relative to the models directory, or to its pool + Path string `json:"path"` // relative to the models directory, or to its pool + Engine string `json:"engine,omitempty"` + // Context is the context length that an Edge0 folder declares. A GGUF + // model's context is in its file metadata. + Context int `json:"context,omitempty"` + // RAM is the system memory that an Edge0 model keeps while it runs on + // one GPU. Edge0RAMOn gives the lower figure of a model that spans + // GPUs. A GGUF model runs from VRAM, and it has no value here. + RAM int64 `json:"ram_bytes,omitempty"` Pool string `json:"pool,omitempty"` // empty for the models directory Size int64 `json:"size_bytes"` Shards int `json:"shards"` @@ -32,6 +46,52 @@ type Model struct { Drafter bool `json:"drafter,omitempty"` } +// Edge0DefaultContext is the context that an Edge0 model runs with when its +// settings name none. It is the value that the model's own launch +// configuration uses, and the value the engine was measured with. +const Edge0DefaultContext = 131072 + +// Edge0RAMOverhead is the system memory that the Edge0 engine keeps beside +// the expert file. On the test host, Edge0 145B with a 37.3 GiB expert file +// took 41 GiB of RAM. +const Edge0RAMOverhead = 6 << 30 + +// Edge0RAM returns the system memory that an Edge0 model keeps while it +// runs on one GPU, for the size of its expert file. The engine holds the +// whole file in RAM and copies experts from there to the GPU. +func Edge0RAM(experts int64) int64 { return experts + Edge0RAMOverhead } + +// Edge0SpanShare is the part of the expert file that stays in RAM when the +// model spans GPUs. The launcher then starts the engine in its low-RAM +// mode, which keeps in RAM only the experts that no GPU holds. On two 24 GB +// cards that was 21.4 GiB of a 37.3 GiB file, which is 57%, so 65% leaves a +// margin. More cards or larger cards keep less. +const Edge0SpanShare = 0.65 + +// Edge0RAMOn returns the system memory that the model keeps while it runs +// on the given number of GPUs. A GGUF model has none. +func (m Model) Edge0RAMOn(gpus int) int64 { + if m.RAM == 0 || gpus < 2 { + return m.RAM + } + experts := m.RAM - Edge0RAMOverhead + return int64(float64(experts)*Edge0SpanShare) + Edge0RAMOverhead +} + +// Edge0Context returns the context that an Edge0 model runs with for the +// ctx_size setting: the setting, or the default, and never more than the +// folder declares. +func (m Model) Edge0Context(ctxSize int) int { + ctx := ctxSize + if ctx <= 0 { + ctx = Edge0DefaultContext + } + if m.Context > 0 && ctx > m.Context { + ctx = m.Context + } + return ctx +} + // PoolRE matches the name of a model pool directory. var PoolRE = regexp.MustCompile(`^pool-[0-9]{1,2}$`) @@ -111,7 +171,25 @@ func Scan(dir string) ([]Model, error) { if path != dir && strings.HasPrefix(d.Name(), ".") { return filepath.SkipDir // partial downloads live in hidden directories } - return nil + if path == dir { + return nil + } + ctx, experts, ok := edge0Folder(path) + if !ok { + return nil + } + // The folder is one model. Its GGUF files are parts of it. + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + size, err := dirSize(path) + if err != nil { + return err + } + id := ID(rel) + byID[id] = &Model{ID: id, Path: rel, Engine: EngineEdge0, Context: ctx, RAM: Edge0RAM(experts), Size: size, Shards: 1} + return filepath.SkipDir } name := d.Name() if !strings.HasSuffix(strings.ToLower(name), ".gguf") || strings.HasPrefix(strings.ToLower(name), "mmproj") { @@ -158,6 +236,54 @@ func Scan(dir string) ([]Model, error) { return out, nil } +// edge0Manifest is the part of an Edge0 folder's model.json that Sibyl reads. +type edge0Manifest struct { + Family string `json:"family"` + Context int `json:"context"` +} + +// edge0Folder reports whether dir is an Edge0 model folder. It returns the +// context length that the folder's model.json declares, and the size of the +// expert file. The folder holds a strata pack: the expert file, and a +// model.json that names the Edge0 family. +func edge0Folder(dir string) (ctx int, experts int64, ok bool) { + path := filepath.Join(dir, "model.json") + // The manifest is about 2 KB. A large file is not one. + if fi, err := os.Stat(path); err != nil || fi.Size() > 1<<20 { + return 0, 0, false + } + raw, err := os.ReadFile(path) + if err != nil { + return 0, 0, false + } + var m edge0Manifest + if err := json.Unmarshal(raw, &m); err != nil || m.Family != "Edge0" { + return 0, 0, false + } + fi, err := os.Stat(filepath.Join(dir, "experts.bin")) + if err != nil { + return 0, 0, false + } + return m.Context, fi.Size(), true +} + +// dirSize returns the total size of the regular files under dir. +func dirSize(dir string) (int64, error) { + var total int64 + err := filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + info, err := d.Info() + if err != nil { + return err + } + total += info.Size() + return nil + }) + return total, err +} + // ID turns a relative path into a model ID that is safe in URLs and llama-swap config. // A file in a folder of the same name, such as qwen/Qwen.gguf, gets the short ID "qwen". func ID(rel string) string { diff --git a/internal/models/models_test.go b/internal/models/models_test.go index 0caa64a..bba726c 100644 --- a/internal/models/models_test.go +++ b/internal/models/models_test.go @@ -148,3 +148,67 @@ func TestLoadSettingsMovesBatchArgs(t *testing.T) { t.Errorf("b keeps its typed batch: %+v", b) } } + +func TestScanEdge0Folder(t *testing.T) { + dir := t.TempDir() + folder := filepath.Join(dir, "Edge0-145B-q3k") + touch(t, filepath.Join(folder, "experts.bin"), 40) + touch(t, filepath.Join(folder, "model-skeleton.gguf"), 6) + touch(t, filepath.Join(folder, "ple-fp8.gguf"), 51) + touch(t, filepath.Join(folder, "mtp", "experts.bin"), 2) + if err := os.WriteFile(filepath.Join(folder, "model.json"), []byte(`{"family":"Edge0","context":262144}`), 0o644); err != nil { + t.Fatal(err) + } + // A folder with a model.json of another family stays a GGUF folder. + touch(t, filepath.Join(dir, "other", "Other.gguf"), 3) + if err := os.WriteFile(filepath.Join(dir, "other", "model.json"), []byte(`{"family":"X"}`), 0o644); err != nil { + t.Fatal(err) + } + + got, err := Scan(dir) + if err != nil { + t.Fatal(err) + } + manifest := int64(len(`{"family":"Edge0","context":262144}`)) + want := []Model{ + {ID: "edge0-145b-q3k", Path: "Edge0-145B-q3k", Engine: EngineEdge0, Context: 262144, RAM: 40 + Edge0RAMOverhead, Size: 99 + manifest, Shards: 1}, + {ID: "other", Path: filepath.Join("other", "Other.gguf"), Size: 3, Shards: 1}, + } + if len(got) != len(want) { + t.Fatalf("got %+v, want %+v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("model %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func TestEdge0Context(t *testing.T) { + m := Model{Engine: EngineEdge0, Context: 262144} + for _, c := range []struct{ setting, want int }{ + {0, Edge0DefaultContext}, {65536, 65536}, {262144, 262144}, {500000, 262144}, + } { + if got := m.Edge0Context(c.setting); got != c.want { + t.Errorf("Edge0Context(%d) = %d, want %d", c.setting, got, c.want) + } + } + if got := (Model{Context: 32768}).Edge0Context(0); got != 32768 { + t.Errorf("a folder with a small context: got %d, want 32768", got) + } +} + +func TestEdge0RAMOn(t *testing.T) { + const experts = 40 << 30 + m := Model{Engine: EngineEdge0, RAM: Edge0RAM(experts)} + if got := m.Edge0RAMOn(1); got != experts+Edge0RAMOverhead { + t.Errorf("one GPU: %d", got) + } + // 65% of 40 GiB is 26 GiB, plus the overhead. + if got, want := m.Edge0RAMOn(2), int64(26<<30)+Edge0RAMOverhead; got != want { + t.Errorf("two GPUs: %d, want %d", got, want) + } + if got := (Model{}).Edge0RAMOn(2); got != 0 { + t.Errorf("a GGUF model: %d", got) + } +} diff --git a/internal/tui/download.go b/internal/tui/download.go index 95547e2..d836dc5 100644 --- a/internal/tui/download.go +++ b/internal/tui/download.go @@ -15,6 +15,7 @@ import ( "github.com/diffsec/sibyl/internal/download" "github.com/diffsec/sibyl/internal/hf" "github.com/diffsec/sibyl/internal/hostd" + "github.com/diffsec/sibyl/internal/models" ) type searchResult struct { @@ -388,6 +389,10 @@ func (v *downloadView) confirmForm(variant client.Variant) tea.Cmd { v.confirm = true desc := fmt.Sprintf("%s in %d file(s). Fit: %s. Usable VRAM %s, RAM %s.\nCommit %s.", gib(variant.Size), variant.Shards, fitText(variant.Fit), gib(v.repo.VRAM), gib(v.repo.RAM), v.repo.Revision[:12]) + if variant.Engine == models.EngineEdge0 { + desc = fmt.Sprintf("Edge0 model folder: %s in %d file(s). It keeps %s of system RAM while it runs, and this host has %s.\nCommit %s.", + gib(variant.Size), variant.Shards, gib(variant.RAM), gib(v.repo.RAM), v.repo.Revision[:12]) + } fields := []huh.Field{} v.pool = "" where := []huh.Option[string]{huh.NewOption("Models folder", "")} diff --git a/internal/tui/formsections.go b/internal/tui/formsections.go index c201ba7..32277d7 100644 --- a/internal/tui/formsections.go +++ b/internal/tui/formsections.go @@ -10,6 +10,7 @@ import ( "github.com/charmbracelet/x/ansi" "github.com/diffsec/sibyl/internal/client" + "github.com/diffsec/sibyl/internal/models" ) // A long settings form shows one group at a time. The section bar above it @@ -32,10 +33,19 @@ var settingsSections = []settingsSection{ {"Advanced", []string{"Args"}}, } +// edge0Sections are the settings that apply to an Edge0 model. Its engine +// is not llama.cpp, so the llama-server settings have no effect on it. +var edge0Sections = []settingsSection{ + {"Model", []string{"Layout", "CtxSize", "Hidden"}}, +} + // visibleSections returns the sections that the form shows for m. The // form hides the experts group for a dense model, and the reasoning group // for a template with no reasoning control. func visibleSections(m client.Model) []settingsSection { + if m.Engine == models.EngineEdge0 { + return edge0Sections + } var out []settingsSection for _, s := range settingsSections { switch s.title {