Run M.O.G.-SEC-27B-1M-CTX on Modal: fp8 KV cache + deployment

README.md

@@ -21,7 +21,7 @@ FreeToken is an edge-native Mixture-of-Experts (MoE) serving engine designed for
 - **Semantic-Aware Caching**: Features semantic anchor checkpoints for recurrent state and KV caches, allowing agentic context edits (e.g., tool calls, thinking blocks) to avoid redundant context recomputation.  
 - **Elastic Memory Management**: Supports dynamic, runtime VRAM re-allocation between expert caches and KV memory without engine restarts or weight reloading.  
 - **Broad MoE & Ecosystem Support**: Supports frontier open-weight MoE models (e.g., DeepSeek-V4-Flash, Qwen3.6-35B-A3B, GLM-5.2) across various parameter scales and quantization formats (e.g., MXFP4, NVFP4, FP8, BF16), with Anthropic/OpenAI-compatible APIs for seamless integration with real-world coding and tool-calling agents (e.g., Codex, Claude Code, OpenCode, OpenClaw, DeepSeek Harness). 
-- **Diverse Consumer Hardware**: Scales across consumer laptops, gaming desktops, and workstation GPUs, with native support for NVIDIA RTX 30, RTX 40, and RTX 50 series GPUs.  
+- **Diverse Consumer Hardware**: Scales across consumer laptops, gaming desktops, and workstation GPUs, with native support for NVIDIA RTX 30, RTX 40, and RTX 50 series GPUs. Intel Arc Pro B60 has an experimental eager PyTorch XPU path for unquantized dense Llama models.
 
 ## Getting Started
 
@@ -55,6 +55,7 @@ For More details:
 - [Quick start](https://github.com/FlashML-org/FreeToken/blob/main/docs/quickstart.md)
 - [Supported models](https://github.com/FlashML-org/FreeToken/blob/main/docs/models.md)
 - [CLI reference](https://github.com/FlashML-org/FreeToken/blob/main/docs/cli.md)
+- [Intel Arc Pro B60](https://github.com/FlashML-org/FreeToken/blob/main/docs/intel-arc-pro-b60.md)
 
 ## Citation
 

deploy/capacity.py

@@ -0,0 +1,186 @@
+"""Decode-throughput / cost model for M.O.G.-SEC-27B-1M-CTX-NVFP4 on Modal.
+
+Autoregressive decode at batch size 1 is memory-bandwidth bound, not compute bound: to
+emit one token the GPU must stream every active weight plus the whole KV cache through
+the SMs exactly once. So
+
+    tokens/sec  ~=  achievable_HBM_bandwidth / bytes_read_per_token
+
+and the engineering problem is entirely "shrink bytes_read_per_token". This module keeps
+that arithmetic in one auditable place; `modal run deploy/modal_app.py::plan` prints it.
+
+Numbers for the model come from the checkpoint's own safetensors headers (measured, not
+guessed) and its config.json.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+GB = 1e9
+
+# --- checkpoint facts (measured from the safetensors headers) ----------------------
+BYTES_TOTAL = 29.390_348_384 * GB
+BYTES_VISION = 0.921 * GB          # SigLIP-style tower; dead weight for text-only serving
+BYTES_LM_HEAD = 2.543 * GB         # [248320, 5120] bf16
+BYTES_EMBED = 2.543 * GB           # not read during decode (it is a gather, not a matmul)
+BYTES_LINEAR_ATTN = 11.124 * GB    # 48 GatedDeltaNet layers, left unquantized by the release
+BYTES_MLP = 9.626 * GB             # 64 layers, NVFP4
+BYTES_SELF_ATTN = 2.632 * GB       # 16 full-attention layers; q/k/v bf16, o_proj NVFP4
+
+# --- architecture (config.json) ----------------------------------------------------
+N_LAYERS = 64
+N_FULL_ATTN = 16                   # layer_types: every 4th layer
+N_LINEAR_ATTN = 48
+N_KV_HEADS = 4
+HEAD_DIM = 256
+LIN_V_HEADS, LIN_V_DIM, LIN_K_HEADS, LIN_K_DIM = 48, 128, 16, 128
+CONV_KERNEL = 4
+
+
+@dataclass(frozen=True)
+class Gpu:
+    name: str
+    mem_gb: float
+    bw_tbs: float          # peak HBM bandwidth, TB/s
+    usd_per_hour: float
+
+
+# Modal published pricing (per-second x 3600).
+GPUS = [
+    Gpu("L40S",          48,  0.864, 0.000542 * 3600),
+    Gpu("A100-80GB",     80,  2.039, 0.000694 * 3600),
+    Gpu("RTX PRO 6000",  96,  1.792, 0.000842 * 3600),
+    Gpu("H100 SXM5",     80,  3.350, 0.001097 * 3600),
+    Gpu("H200 SXM",     141,  4.800, 0.001261 * 3600),
+    Gpu("B200",         180,  8.000, 0.001736 * 3600),
+    Gpu("B300",         288,  8.000, 0.001972 * 3600),
+]
+
+# Fraction of peak HBM bandwidth a well-tuned decode loop actually sustains. 0.80 is
+# optimistic-but-real for a fused CUDA-graph decode; 0.70 is the conservative planning
+# number used for the headline claim.
+BW_EFFICIENCY = 0.70
+
+
+def kv_bytes_per_token(kv_bits: int) -> float:
+    """KV bytes per token, counting only the 16 full-attention layers."""
+    return 2 * N_KV_HEADS * HEAD_DIM * N_FULL_ATTN * (kv_bits / 8)
+
+
+def linear_state_bytes(ssm_bits: int = 32) -> float:
+    """GatedDeltaNet recurrent state. Constant in context length -- the whole point."""
+    per_layer_ssm = LIN_V_HEADS * LIN_V_DIM * LIN_K_DIM * (ssm_bits / 8)
+    conv_dim = 2 * LIN_K_HEADS * LIN_K_DIM + LIN_V_HEADS * LIN_V_DIM
+    per_layer_conv = conv_dim * CONV_KERNEL * 2
+    return N_LINEAR_ATTN * (per_layer_ssm + per_layer_conv)
+
+
+def weight_bytes(*, drop_vision: bool = True, lm_head_bits: int = 16,
+                 linear_attn_bits: int = 16) -> float:
+    """Active weight bytes streamed per decode step. The model is dense, so 'active'
+    means all of them -- there is no MoE sparsity to exploit here."""
+    total = BYTES_TOTAL
+    if drop_vision:
+        total -= BYTES_VISION
+    total -= BYTES_EMBED                      # embedding lookup is a gather, not a stream
+    total -= BYTES_LM_HEAD
+    total += BYTES_LM_HEAD * (lm_head_bits / 16)
+    total -= BYTES_LINEAR_ATTN
+    total += BYTES_LINEAR_ATTN * (linear_attn_bits / 16)
+    return total
+
+
+def bytes_per_token(ctx: int, *, kv_bits: int = 8, lm_head_bits: int = 16,
+                    linear_attn_bits: int = 16, drop_vision: bool = True) -> dict:
+    w = weight_bytes(drop_vision=drop_vision, lm_head_bits=lm_head_bits,
+                     linear_attn_bits=linear_attn_bits)
+    kv = kv_bytes_per_token(kv_bits) * ctx
+    st = linear_state_bytes()
+    return {"weights": w, "kv": kv, "state": st, "total": w + kv + st}
+
+
+def tps(gpu: Gpu, ctx: int, *, tp: int = 1, efficiency: float = BW_EFFICIENCY,
+        tp_scaling: float = 0.85, **kw) -> float:
+    """Predicted decode tok/s. With tensor parallelism each GPU reads its own shard, so
+    per-GPU bytes fall ~linearly while collectives eat `tp_scaling` of the win."""
+    b = bytes_per_token(ctx, **kw)["total"] / tp
+    eff_bw = gpu.bw_tbs * 1e12 * efficiency * (tp_scaling if tp > 1 else 1.0)
+    return eff_bw / b
+
+
+def footprint_gb(ctx: int, *, tp: int = 1, workspace_gb: float = 4.0, **kw) -> float:
+    """Resident VRAM per GPU: weights + KV + state + activation/workspace slack."""
+    b = bytes_per_token(ctx, **kw)
+    resident = b["weights"] + b["kv"] + b["state"] + BYTES_EMBED
+    return resident / tp / GB + workspace_gb
+
+
+def report(ctx: int = 1_000_000, target_tps: float = 100.0) -> str:
+    lines: list[str] = []
+    A = lines.append
+    A(f"M.O.G.-SEC-27B-1M-CTX-NVFP4 decode model @ {ctx:,} ctx (target {target_tps:.0f} tok/s)")
+    A("=" * 94)
+    A("")
+    A("Hybrid architecture is what makes this tractable:")
+    A(f"  {N_FULL_ATTN} full-attention layers carry the KV cache; {N_LINEAR_ATTN} GatedDeltaNet layers")
+    A(f"  hold a fixed {linear_state_bytes()/GB:.3f} GB state regardless of context length.")
+    A(f"  A same-size all-full-attention model would need {N_LAYERS/N_FULL_ATTN:.0f}x the KV bandwidth.")
+    A("")
+
+    configs = [
+        ("bf16 KV (today)", dict(kv_bits=16)),
+        ("fp8 KV", dict(kv_bits=8)),
+        ("fp8 KV+head", dict(kv_bits=8, lm_head_bits=8)),
+        ("fp8 all", dict(kv_bits=8, lm_head_bits=8, linear_attn_bits=8)),
+        ("fp4 KV/fp8 all", dict(kv_bits=4, lm_head_bits=8, linear_attn_bits=8)),
+    ]
+    A(f"{'configuration':<20}{'weights':>10}{'KV@1M':>10}{'total':>10}   (bytes read per decode token)")
+    A("-" * 94)
+    for name, kw in configs:
+        b = bytes_per_token(ctx, **kw)
+        A(f"{name:<20}{b['weights']/GB:9.1f}G{b['kv']/GB:9.1f}G{b['total']/GB:9.1f}G")
+    A("")
+
+    A(f"Predicted decode tok/s (at {BW_EFFICIENCY:.0%} of peak HBM bandwidth):")
+    A("")
+    hdr = f"{'GPU':<15}{'$/hr':>7}{'mem':>6}"
+    A(hdr + "".join(f"{n:>16}" for n, _ in configs))
+    A("-" * 94)
+    for g in GPUS:
+        row = f"{g.name:<15}{g.usd_per_hour:7.2f}{g.mem_gb:5.0f}G"
+        for _name, kw in configs:
+            t = tps(g, ctx, **kw)
+            fits = footprint_gb(ctx, **kw) <= g.mem_gb
+            row += f"{(f'{t:.0f}' if fits else 'OOM'):>16}"
+        A(row)
+    A("")
+
+    # Cheapest configuration that clears the target.
+    A(f"Cheapest options clearing {target_tps:.0f} tok/s:")
+    A("")
+    winners = []
+    for g in GPUS:
+        for tp in (1, 2, 4):
+            for name, kw in configs:
+                t = tps(g, ctx, tp=tp, **kw)
+                if t >= target_tps and footprint_gb(ctx, tp=tp, **kw) <= g.mem_gb:
+                    winners.append((g.usd_per_hour * tp, g.name, tp, name, t))
+    winners.sort()
+    seen = set()
+    for cost, gname, tp, cname, t in winners:
+        if (gname, tp) in seen:
+            continue
+        seen.add((gname, tp))
+        per_mtok = cost / (t * 3600) * 1e6
+        A(f"  ${cost:6.2f}/hr  {gname:<14} TP={tp}  {cname:<28} {t:6.0f} tok/s  "
+          f"(${per_mtok:.2f}/M output tok)")
+        if len(seen) >= 6:
+            break
+    if not winners:
+        A("  none -- need a bigger lever (speculative decoding, or TP>4)")
+    return "\n".join(lines)
+
+
+if __name__ == "__main__":
+    print(report())