{ "hf_id": "thinkingmachines/Inkling", "meta": { "title": "TML Inkling", "slug": "tml-inkling", "provider": "Thinking Machines Lab", "description": "Natively multimodal 1T-parameter MoE from Thinking Machines Lab — text/image/audio in, text out, up to 1M context — with relative attention, short convolution, and shared expert sinks.", "date_added": "2026-07-15", "date_updated": "2026-08-04", "tasks": [ "multimodal" ], "performance_headline": "380 tok/s/user with MTP8 (mean acceptance length 4.5) and 140 tok/s/user without MTP on 4× GB200 GPUs", "related_recipes": [], "hardware": { "gb200": "verified", "b200": "verified", "h200": "verified", "mi300x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_USE_V2_MODEL_RUNNER": "1", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" }, "docker_image": "vllm/vllm-openai:nightly", "command": "vllm serve thinkingmachines/Inkling-NVFP4 \\\n --trust-remote-code \\\n --tokenizer-mode inkling \\\n --disable-flashinfer-autotune \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser inkling \\\n --reasoning-parser inkling", "argv": [ "vllm", "serve", "thinkingmachines/Inkling-NVFP4", "--trust-remote-code", "--tokenizer-mode", "inkling", "--disable-flashinfer-autotune", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "inkling", "--reasoning-parser", "inkling" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED=1 \\\n vllm/vllm-openai:nightly thinkingmachines/Inkling-NVFP4 \\\n --trust-remote-code \\\n --tokenizer-mode inkling \\\n --disable-flashinfer-autotune \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser inkling \\\n --reasoning-parser inkling", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED=1", "vllm/vllm-openai:nightly", "thinkingmachines/Inkling-NVFP4", "--trust-remote-code", "--tokenizer-mode", "inkling", "--disable-flashinfer-autotune", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "inkling", "--reasoning-parser", "inkling" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/thinkingmachines/Inkling/strategies/single_node_tep.json", "multi_node_tp_dp": "/thinkingmachines/Inkling/strategies/multi_node_tp_dp.json", "kv_store_centralized_mooncake": "/thinkingmachines/Inkling/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/thinkingmachines/Inkling/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/thinkingmachines/Inkling/hw/h200.json", "h100": "/thinkingmachines/Inkling/hw/h100.json", "b200": "/thinkingmachines/Inkling/hw/b200.json", "gb200": "/thinkingmachines/Inkling/hw/gb200.json", "b300": "/thinkingmachines/Inkling/hw/b300.json", "gb300": "/thinkingmachines/Inkling/hw/gb300.json" } }, "model": { "model_id": "thinkingmachines/Inkling-NVFP4", "min_vllm_version": "0.26.0", "nightly_required": true, "architecture": "moe", "parameter_count": "975B", "active_parameters": "41B", "context_length": 1048576, "base_args": [ "--trust-remote-code", "--tokenizer-mode", "inkling" ], "base_env": { "VLLM_USE_V2_MODEL_RUNNER": "1" }, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --pre \\\n --extra-index-url https://wheels.vllm.ai/nightly/cu130 \\\n --extra-index-url https://download.pytorch.org/whl/cu130 \\\n --index-strategy unsafe-best-match", "note": "vLLM 0.26.0 isn't released yet — nightly required. Nightly wheels are CUDA 13 (cu130) only; the host needs an r580+ NVIDIA driver." }, "docker": { "command": "docker pull vllm/vllm-openai:latest" }, "extras": [ { "note": "Audio extras — only needed when serving the audio modality", "command": "uv pip install \"vllm[audio]\"", "optional": true } ] } }, "features": { "tool_calling": { "description": "Enable auto tool choice with the Inkling tool-call parser.", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "inkling" ] }, "reasoning": { "description": "Enable reasoning output parsing with the Inkling reasoning parser.", "args": [ "--reasoning-parser", "inkling" ] }, "spec_decoding": { "description": "Multi-Token Prediction (MTP) speculative decoding. Inkling ships 8 chained MTP heads, allowing up to 9 tokens per forward step; tune num_speculative_tokens for your acceptance/latency trade-off.", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":8}" ], "hardware_overrides": { "hopper": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" ] } } } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "nvfp4", "precision_hardware": { "brand": "NVIDIA" }, "vram_minimum_gb": 651, "description": "Weight-only NVFP4 (group size 16) on the routed MoE expert matmuls only; attention, norms, sconv, shared experts, routers, embeddings, and the layer 0-1 dense MLPs stay BF16. ~542 GB on disk. On Hopper, the MoE kernel dequantizes the NVFP4 expert weights to BF16 on the fly.", "extra_args": [ "--disable-flashinfer-autotune" ], "extra_env": { "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "mxfp4": { "model_id": "lightseekorg/Inkling-MXFP4", "precision": "mxfp4", "precision_hardware": { "brand": "AMD" }, "vram_minimum_gb": 540, "supported_hardware": [ "mi300x", "mi325x", "mi355x" ], "tp": { "mi300x": 4, "mi325x": 4, "mi355x": 4 }, "description": "Quark OCP MXFP4 checkpoint for AMD Instinct: TP4 on MI355X with native AITER MXFP4 MoE, or TP4 on MI300X/MI325X with Triton MXFP4 emulation. Full 1M context is validated on MI300X and MI355X; MI325X uses the same gfx942 path but is not yet smoke-tested.", "hardware_overrides": { "mi300x": { "docker_image": "vllm/vllm-openai-rocm:nightly", "extra_args": [ "--moe-backend", "auto", "--block-size", "128", "--gpu-memory-utilization", "0.95", "--enforce-eager" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "INKLING_GFX950_GLUON": "0" } }, "mi325x": { "docker_image": "vllm/vllm-openai-rocm:nightly", "extra_args": [ "--moe-backend", "auto", "--block-size", "128", "--gpu-memory-utilization", "0.95", "--enforce-eager" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "INKLING_GFX950_GLUON": "0" } }, "mi355x": { "docker_image": "vllm/vllm-openai-rocm:nightly", "extra_args": [ "--moe-backend", "aiter", "--block-size", "128", "--gpu-memory-utilization", "0.90" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "INKLING_GFX950_GLUON": "0" } } }, "json": "/lightseekorg/Inkling-MXFP4.json" }, "bf16": { "model_id": "thinkingmachines/Inkling", "precision": "bf16", "precision_hardware": { "brand": "NVIDIA" }, "vram_minimum_gb": 2400, "description": "Full BF16 checkpoint (~1.8 TB on disk). Runs on Hopper and Blackwell.", "extra_args": [ "--disable-flashinfer-autotune" ], "extra_env": { "FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED": "1" } }, "fp8": { "model_id": "RedHatAI/Inkling-FP8-dynamic", "precision": "fp8", "vram_minimum_gb": 1170, "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Inkling-FP8-dynamic.json" }, "redhat_fp8": { "model_id": "RedHatAI/Inkling-FP8-block", "precision": "fp8", "label": "FP8 Block (RedHat)", "vram_minimum_gb": 1170, "description": "RedHatAI FP8 block quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Inkling-FP8-block.json" }, "redhat_nvfp4": { "model_id": "RedHatAI/Inkling-NVFP4", "precision": "nvfp4", "label": "NVFP4 (RedHat)", "vram_minimum_gb": 618, "description": "RedHatAI NVFP4 quantization", "json": "/RedHatAI/Inkling-NVFP4.json" } }, "strategy_hardware": { "multi_node_tp": { "hopper": "unsupported" }, "multi_node_tep": { "hopper": "unsupported" }, "multi_node_dep": { "hopper": "unsupported" }, "pd_cluster": { "hopper": "unsupported" } }, "hardware_overrides": {}, "guide": "## Overview\n\nTML Inkling is a natively multimodal 1T-parameter Mixture-of-Experts model from Thinking\nMachines Lab. It accepts text, image, and audio inputs and generates text, with a context\nlength of up to 1M tokens. vLLM supports Inkling on Day 0 with full feature parity — LoRA,\nspeculative decoding (MTP), TP/DP/EP/PP parallelism, prefix caching, and disaggregated serving.\n\nArchitectural highlights:\n\n- **Attention.** 66-layer decoder backbone — 11 full-attention layers and 55 sliding-window\n layers — all grouped-query attention with head size 128. Inkling replaces RoPE with\n **relative attention**: a learned relative-position term added to the pre-softmax logits.\n- **Short convolution (sconv).** Each layer applies four window-4 sconv modules (on attention\n keys, values, output, and MoE output). vLLM manages the sconv cache as the KV cache of a\n virtual sliding-window layer, so prefix caching works seamlessly, and shards sconv across\n the channel dimension (reduce-scatter / all-gather) to avoid replicating cache and compute\n across TP ranks.\n- **MoE with expert sinks.** 256 routed experts (top-6) plus 2 shared experts — 8 experts per\n token. The 2 shared experts act as *expert sinks*: they participate in routing-score\n computation but are excluded from the top-6 candidate pool.\n- **MTP.** 8 chained MTP heads (single-layer full-attention transformers with a dense MLP,\n all BF16) enable up to 9 tokens per forward step for speculative decoding.\n\n## Variants\n\n- **NVFP4 (default):** Only the routed experts are quantized to NVFP4; shared experts and the\n qkvr linears remain BF16. This is a mixed-precision checkpoint. Blackwell uses its native\n FP4 tensor cores; on Hopper, the MoE kernel dequantizes the NVFP4 expert weights to BF16\n on the fly.\n- **BF16:** Full BF16 weights (`thinkingmachines/Inkling`). Runs on both **Hopper**\n (H200&H20) and Blackwell, at the cost of substantially more VRAM.\n- **MXFP4 (AMD):** `lightseekorg/Inkling-MXFP4` keeps the full 1M-token context\n at TP4 on MI355X and MI300X. MI355X runs native AITER MXFP4 MoE; MI300X uses\n Triton on-the-fly MXFP4 emulation.\n\n## Prerequisites\n\n- **Hardware:** NVIDIA Hopper (H200/H20) or Blackwell for NVFP4 and BF16;\n AMD MI300X, MI325X, or MI355X with TP4 for MXFP4.\n- **vLLM:** Nightly wheels (Day-0 support just landed).\n- **Parallelism:** 1T parameters requires multi-GPU. The featured configuration is 4x GB200\n (`--tensor-parallel-size 4`) for the NVFP4 variant. On Hopper, NVFP4 uses TP8 on H200;\n the BF16 variant needs multi-node.\n\n## Client Usage\n\nLaunch the server (NVFP4 on 4x GB200):\n```bash\nexport VLLM_USE_V2_MODEL_RUNNER=1\nexport FLASH_ATTENTION_CUTE_DSL_CACHE_ENABLED=1\n\nvllm serve thinkingmachines/Inkling-NVFP4 \\\n --tokenizer-mode inkling \\\n --reasoning-parser inkling \\\n --tool-call-parser inkling \\\n --enable-auto-tool-choice \\\n --tensor-parallel-size 4 \\\n --disable-flashinfer-autotune \\\n --trust-remote-code\n```\n\nFor a multi-node BF16 deployment on Hopper (H200/H20), select the\n**Multi-Node TP + Data Parallel** strategy with **Nodes = 2** in the command\nbuilder above — it renders the per-node `vllm serve` commands (TP within\neach node, DP across nodes, EP over all ranks).\n\nOn AMD, select the **MXFP4** variant. TP4 is validated with the native\n1M context on both GPU families.\n\nEquivalent MI355X launch:\n```bash\nexport VLLM_USE_V2_MODEL_RUNNER=1\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_USE_AITER_MOE=1\nexport INKLING_GFX950_GLUON=0\n\nvllm serve lightseekorg/Inkling-MXFP4 \\\n --tensor-parallel-size 4 \\\n --tokenizer-mode inkling \\\n --reasoning-parser inkling \\\n --tool-call-parser inkling \\\n --enable-auto-tool-choice \\\n --moe-backend aiter \\\n --block-size 128 \\\n --trust-remote-code\n```\n\nQuery the server with the OpenAI SDK, including an image input:\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\n\nresponse = client.chat.completions.create(\n model=\"thinkingmachines/Inkling-NVFP4\",\n messages=[{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"text\", \"text\": \"Describe this image.\"},\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://example.com/photo.jpg\"}},\n ],\n }],\n)\nprint(response.choices[0].message.content)\n```\n\n## Performance\n\nvLLM implements Inkling with a series of optimizations: sconv-aware TP sharding (each rank\nstores only its channel shard of the sconv cache), low-latency fused reduce-scatter /\nall-gather collectives built on FlashInfer's Lamport-protocol all-reduce, the FA4\nsheared-bias attention kernel for relative attention, plus kernel fusion, PDL, and\nmulti-streaming. Enabling MTP speculative decoding adds further per-user throughput.\n\n## References\n\n- [TML Inkling (NVFP4)](https://huggingface.co/thinkingmachines/Inkling-NVFP4)\n- [TML Inkling (BF16)](https://huggingface.co/thinkingmachines/Inkling)\n- [Inkling MXFP4](https://huggingface.co/lightseekorg/Inkling-MXFP4)\n- [vLLM integration PR](https://github.com/Inferact/mlj/pull/16)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/thinkingmachines/Inkling.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/thinkingmachines/Inkling.yaml", "overlays": [] } }