feat(gguf): serve deepseek-v4.1 gguf checkpoints on deepseek_v41

docs/models.md

@@ -153,6 +153,15 @@ unset on PCIe GPUs: wider grids add no bandwidth there.
   `deepseek4v` mmproj; the text-only 0731 release has no image token and refuses the
   mmproj. ggml-org's Vision-Exp text GGUF ships a chat template that renders no image
   placeholder; serve unsloth's release for image requests.
+- A DeepSeek-V4.1 GGUF (llama.cpp's `deepseek41` layout, such as
+  [unsloth/DeepSeek-V4.1-Flash-1017-GGUF](https://huggingface.co/unsloth/DeepSeek-V4.1-Flash-1017-GGUF))
+  serves text; its `deepseek41v` mmproj and DSpark draft are not read. The Engram tables
+  stream from the file's `per_layer_token_embd` tensor, so keep that file on a fast NVMe.
+  A build that ships the tables as a separate shard needs it linked next to the others under
+  its shard name, as llama.cpp does. A file converted for an earlier llama.cpp branch has
+  no `ple.layers` key and is refused, and so are unsloth's UD-Q3_K_XL, UD-Q4_K_XL and UD-Q8_K_XL,
+  whose MXFP8 tables only unsloth's llama.cpp PR reads. `ft checkpoint` does not convert these
+  GGUFs: serve the `.gguf`.
 - DeepSeek-V4.1-Flash serves text and images from the HF checkpoint as shipped (fp8 block-32
   dense weights, fp4 experts on the offload cache, 890 B/token global KV in packed
   fp4/fp8 pools). Its two 98 GiB Engram tables stream from the checkpoint shards on

python/freetoken/checkpoint/convert.py

@@ -214,6 +214,16 @@ def convert_checkpoint(
 
     from freetoken.utils.progress import byte_bar, count_bar
 
+    # a family refuses a source it has no FTW form for before the writer creates out_dir
+    from freetoken.models.register import _load_attr, get_model_spec
+
+    try:
+        check_source = _load_attr(get_model_spec(mc.architectures[0]).module, "ftw_check_source")
+    except (AttributeError, KeyError, ValueError):
+        check_source = None
+    if check_source is not None:
+        check_source(model_path)
+
     writer = FTWWriter(out_dir, shard_limit=shard_limit)
     n_weight = n_bank = n_alpha = 0
 
@@ -233,8 +243,6 @@ def convert_checkpoint(
         _progress("dense", dense_bytes, 0)
 
     # files the model reads directly from the checkpoint dir, not through FTW entries (Qwen3.8-Flash-Next PLE table)
-    from freetoken.models.register import _load_attr, get_model_spec
-
     try:
         side_hook = _load_attr(get_model_spec(mc.architectures[0]).module, "ftw_side_files")
     except (AttributeError, KeyError, ValueError):

python/freetoken/kernel/aot_models.py

@@ -343,6 +343,8 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
         # the routed experts share DSV4's ds_fp4 bank layout (hidden 5120, intermediate 2304).
         name="deepseek-ai/DeepSeek-V4.1-Flash",
         architecture="DeepseekV41ForCausalLM",
+        # llama.cpp deepseek41 GGUF (unsloth 1017 GGUF; text only)
+        arch_aliases=("DeepseekV41GGUFForCausalLM",),
         hidden_size=5120,
         kv_groups=(),
         top_k=6,

python/freetoken/models/deepseek_v4/gguf.py

@@ -31,10 +31,10 @@
 import numpy as np
 import torch
 
-from freetoken.layers.quantization.ggml_types import is_quantized
+from freetoken.models.gguf.config import routed_expert_count, uniform_swiglu_clamp
 from freetoken.models.gguf.mmproj import Mmproj, find_mmproj, log_mmproj, mmproj_tensor
 from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, gguf_tensor_names, load_gguf_metadata
-from freetoken.models.gguf.tensors import check_scales, fused, matrix
+from freetoken.models.gguf.tensors import check_scales, dequantized_tensor as _float, fused, matrix
 from freetoken.utils import init_logger
 
 from .args import DeepseekV4Args
@@ -127,14 +127,7 @@ def need(key: str):
     # process nulls vision_n_layers on its copy, and None here is the same answer
     vc = parse_vision_config(shim)
 
-    # the clamp is per-layer in the GGUF (and the shared expert carries its own array); the model
-    # takes one value, so anything non-uniform is refused instead of silently taking the first
-    clamp = need("swiglu_clamp_exp")
-    clamp_vals = list(clamp) if isinstance(clamp, list) else [clamp]
-    clamp_shexp = kv("swiglu_clamp_shexp")
-    shexp_vals = list(clamp_shexp) if isinstance(clamp_shexp, list) else [clamp_shexp] if clamp_shexp is not None else []
-    if len(set(map(float, clamp_vals + shexp_vals))) != 1:
-        raise NotImplementedError(f"{shim.model_path}: non-uniform swiglu_clamp_exp / swiglu_clamp_shexp is not served")
+    swiglu_limit = uniform_swiglu_clamp(shim.model_path, need("swiglu_clamp_exp"), kv("swiglu_clamp_shexp"))
 
     from freetoken.third_party.gguf.constants import ExpertGatingFuncType
 
@@ -172,21 +165,8 @@ def need(key: str):
     num_layers = int(need("block_count"))
     index = gguf_tensor_index(shim.model_path)
 
-    # REAP exports prune experts per layer: as a per-layer expert_count array, or with the
-    # scalar kept but the expert tensors shrunk. llama.cpp builds every layer from one
-    # n_expert and refuses both shapes; the model has no per-layer expert count to express.
-    experts_kv = need("expert_count")
-    if isinstance(experts_kv, list):
-        raise NotImplementedError(
-            f"{shim.model_path}: a per-layer expert_count array (REAP-style prune) is not served"
-        )
-    n_experts = int(experts_kv)
-    for name, info in index.items():
-        if name.endswith(".ffn_gate_exps.weight") and info.ne[-1] != n_experts:
-            raise NotImplementedError(
-                f"{shim.model_path}: {name} carries {info.ne[-1]} experts but expert_count says "
-                f"{n_experts} (REAP-style per-layer prunes are not served)"
-            )
+    # llama.cpp builds every layer from one n_expert; the model has no per-layer expert count to express
+    n_experts = routed_expert_count(shim.model_path, need("expert_count"), index)
     args = DeepseekV4Args(
         max_batch_size=1,
         vocab_size=int(shim.vocab_size),
@@ -195,12 +175,12 @@ def need(key: str):
         n_layers=num_layers,
         n_hash_layers=int(kv("hash_layer_count", 0) or 0),
         n_heads=int(need("attention.head_count")),
-        n_routed_experts=int(need("expert_count")),
+        n_routed_experts=n_experts,
         n_shared_experts=int(kv("expert_shared_count", 1) or 1),
         n_activated_experts=int(need("expert_used_count")),
         score_func="sqrtsoftplus",
         route_scale=float(need("expert_weights_scale")),
-        swiglu_limit=float(clamp_vals[0]),
+        swiglu_limit=swiglu_limit,
         q_lora_rank=int(need("attention.q_lora_rank")),
         head_dim=head_dim,
         rope_head_dim=int(need("rope.dimension_count")),
@@ -398,25 +378,6 @@ def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]:
 # Weights
 # ----------------------------------------------------------------------------------------------
 
-def _float(info: GgufTensorInfo) -> torch.Tensor:
-    """A float tensor in torch order, dequantizing packed ggml rows: dynamic quants quantize
-    tensors the module consumes as floats (the mHC mixers, the compressor gates, the router).
-    Trailing unit dims in ``ne`` are dropped -- some writers ship tensors 4-D (llama.cpp's
-    loaders try that shape first) and a trailing 1 carries no elements."""
-    ne = info.ne
-    while len(ne) > 1 and ne[-1] == 1:
-        ne = ne[:-1]
-    if not is_quantized(info.ggml_type):
-        from freetoken.models.gguf.tensors import float_tensor
-
-        return float_tensor(info).view(*reversed(ne))
-    from freetoken.third_party.gguf import quants
-    from freetoken.third_party.gguf.constants import GGMLQuantizationType
-
-    flat = quants.dequantize(np.array(info.data()), GGMLQuantizationType(info.ggml_type))
-    return torch.from_numpy(flat).view(*reversed(ne))
-
-
 def _tid2eid(info: GgufTensorInfo) -> torch.Tensor:
     """The hash-route table as the module's int64 [vocab, top_k]."""
     ne = info.ne

python/freetoken/models/deepseek_v41/__init__.py

@@ -8,7 +8,11 @@
 """
 
 from .config import parse_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
 from .model import DeepseekV41ForCausalLM
-from .weight import ftw_side_files, iter_expert_pieces, iter_weights, iter_vision_weights
+from .weight import ftw_check_source, ftw_side_files, iter_expert_pieces, iter_weights, iter_vision_weights
 
-__all__ = ["DeepseekV41ForCausalLM", "parse_config", "iter_weights", "iter_vision_weights", "iter_expert_pieces", "ftw_side_files"]
+__all__ = [
+    "DeepseekV41ForCausalLM", "parse_config", "iter_weights", "iter_vision_weights", "iter_expert_pieces", "ftw_side_files",
+    "ftw_check_source", "iter_gguf_weights", "parse_gguf_config", "to_gguf",
+]

python/freetoken/models/deepseek_v41/config.py

@@ -31,15 +31,8 @@ class VisionConfig:
     max_wh_ratio: float | None = None
 
 
-def parse_config(hf_config: Any) -> ModelConfig:
-    args = DeepseekV41Args.from_hf(hf_config)
-    text = getattr(hf_config, "text_config", None) or hf_config
-    max_position = int(getattr(text, "max_position_embeddings", 0) or getattr(hf_config, "max_position_embeddings", 0) or 0)
-    if max_position <= 0:
-        max_position = int(args.original_seq_len * args.rope_factor)
-    vc = getattr(hf_config, "vision_config", None)
-    vision = VisionConfig(**{f.name: getattr(vc, f.name, f.default) for f in fields(VisionConfig)}) if vc is not None else None
-
+def model_config_from_args(args: DeepseekV41Args, *, max_position: int, vision: VisionConfig | None = None, image_token_id: int | None = None) -> ModelConfig:
+    """The ModelConfig of a DeepSeek-V4.1 text tower, from the HF config or a GGUF's metadata."""
     return ModelConfig(
         num_layers=args.n_layers,
         num_qo_heads=args.n_heads,
@@ -78,7 +71,7 @@ def parse_config(hf_config: Any) -> ModelConfig:
         attn_sm_scale=args.head_dim**-0.5,
         dsv41_args=args,
         vision_config=vision,
-        image_token_id=getattr(hf_config, "image_token_id", None),
+        image_token_id=image_token_id,
         attention_groups=(
             DSV4AttentionGroupConfig(
                 name="dsv41",
@@ -92,4 +85,15 @@ def parse_config(hf_config: Any) -> ModelConfig:
     )
 
 
-__all__ = ["parse_config"]
+def parse_config(hf_config: Any) -> ModelConfig:
+    args = DeepseekV41Args.from_hf(hf_config)
+    text = getattr(hf_config, "text_config", None) or hf_config
+    max_position = int(getattr(text, "max_position_embeddings", 0) or getattr(hf_config, "max_position_embeddings", 0) or 0)
+    if max_position <= 0:
+        max_position = int(args.original_seq_len * args.rope_factor)
+    vc = getattr(hf_config, "vision_config", None)
+    vision = VisionConfig(**{f.name: getattr(vc, f.name, f.default) for f in fields(VisionConfig)}) if vc is not None else None
+    return model_config_from_args(args, max_position=max_position, vision=vision, image_token_id=getattr(hf_config, "image_token_id", None))
+
+
+__all__ = ["model_config_from_args", "parse_config"]

python/freetoken/models/deepseek_v41/engram_table.py

@@ -66,13 +66,15 @@ def _tensor_extent(folder: str, name: str) -> _TensorExtent:
 
 @dataclass(frozen=True)
 class EngramRowSource:
-    """The fp8 table tensor in its shard (one extent) and its host-resident scales."""
+    """One layer's table in one file extent: fp8 rows with host-resident scales, or a GGUF's rows of ``ggml_type``."""
 
     path: str
     base: int
     num_rows: int
     head_dim: int
-    scales: torch.Tensor  # [num_rows, head_dim // 32] uint8 (e8m0 codes), CPU
+    scales: torch.Tensor | None  # [num_rows, head_dim // 32] uint8 (e8m0 codes), CPU; None for a GGUF table
+    ggml_type: int | None = None
+    row_bytes: int = 0  # a GGUF row's bytes on disk
 
 
 def engram_row_source(folder: str, layer_id: int) -> EngramRowSource:
@@ -105,12 +107,15 @@ def __init__(
         self.head_dim = source.head_dim
         self.n_cols = n_cols
         self.num_rows = source.num_rows
-        self.row_bytes = FP8_E8M0_B32.row_bytes(source.head_dim)  # values | scales
+        self.ggml_type = source.ggml_type
+        # staged rows: values | scales for fp8, the file's bytes for a GGUF
+        self.row_bytes = FP8_E8M0_B32.row_bytes(source.head_dim) if source.ggml_type is None else source.row_bytes
         self.scales = source.scales
         self.device = device
+        stored = source.head_dim if source.ggml_type is None else source.row_bytes
         self.store = RowStore(
             paths=[source.path], extent_file=[0], extent_base=[source.base], rows_per_extent=source.num_rows,
-            row_bytes=source.head_dim, row_stride=source.head_dim, use_io_uring=use_io_uring,
+            row_bytes=stored, row_stride=stored, use_io_uring=use_io_uring,
         )
         # the graph staging is allocated up front (pinned alloc inside capture is illegal) and
         # outlives any one capture so a re-capture keeps the same pointers; the eager staging
@@ -160,15 +165,18 @@ def stage(self, row_ids: torch.Tensor, *, graph: bool) -> None:
         ids = row_ids.reshape(-1).to(torch.int64).contiguous()
         rows = pinned[: n * self.row_bytes].view(n, self.row_bytes)
         self.store.stage_rows(ids.data_ptr(), n, rows.data_ptr(), self.row_bytes)
-        rows[:, self.head_dim :].copy_(self.scales.index_select(0, ids))
+        if self.scales is not None:
+            rows[:, self.head_dim :].copy_(self.scales.index_select(0, ids))
 
     def flush(self, *, signal: bool) -> None:
         self.store.flush(self.flag.data_ptr() if signal else 0)
 
     # ----- device (EngramTable protocol) -----
     def lookup(self, num_tokens: int) -> torch.Tensor:
+        from freetoken.kernel import ggml
         from freetoken.kernel.row_store import wait_reset
         from freetoken.kernel.triton.dsv41.pack import unpack_rows
+        from freetoken.layers.quantization.ggml_types import float_dtype
 
         stream = torch.cuda.current_stream(self.device)
         capturing = torch.cuda.is_current_stream_capturing()
@@ -180,7 +188,14 @@ def lookup(self, num_tokens: int) -> torch.Tensor:
         dev[:nbytes].copy_(pinned[:nbytes], non_blocking=True)
         if not capturing:
             self._eager_read[self._eager_slot].record(stream)
-        values = unpack_rows(dev[:nbytes].view(n, self.row_bytes), FP8_E8M0_B32, self.head_dim)
+        rows = dev[:nbytes].view(n, self.row_bytes)
+        if self.ggml_type is None:
+            values = unpack_rows(rows, FP8_E8M0_B32, self.head_dim)
+        elif (dtype := float_dtype(self.ggml_type)) is not None:
+            # copy=True: bf16 rows must not alias the staging buffer the next copy overwrites
+            values = rows.view(dtype).to(torch.bfloat16, copy=True)
+        else:
+            values = ggml.dequantize(rows, self.ggml_type, self.head_dim)
         return values.view(num_tokens, self.n_cols * self.head_dim)