{ "hf_id": "thinkingmachines/Inkling-2-Small", "meta": { "title": "Inkling 2 Small", "slug": "inkling-2-small", "provider": "Thinking Machines Lab", "description": "Multimodal (text and image in, text out) MoE from Thinking Machines Lab with 594B total and 17B active parameters, up to 1M context, attention residuals, relative-bias attention, short convolutions, and a bundled DSpark speculator.", "date_added": "2026-11-12", "date_updated": "2026-11-12", "difficulty": "intermediate", "tasks": [ "multimodal" ], "performance_headline": "~1,100 output tok/s/GPU at 100 tok/s/user (1K in / 1K out) with DSpark on 2× 4 GB200 (TP8)", "related_recipes": [ "thinkingmachines/Inkling", "thinkingmachines/Inkling-Small" ], "hardware": { "gb200": "verified" } }, "recommended_command": { "hardware": "b200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:inkling-2-small", "command": "vllm serve thinkingmachines/Inkling-2-Small \\\n --tensor-parallel-size 4 \\\n --max-model-len 262144 \\\n --max-num-seqs 256 \\\n --reasoning-parser inkling \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "thinkingmachines/Inkling-2-Small", "--tensor-parallel-size", "4", "--max-model-len", "262144", "--max-num-seqs", "256", "--reasoning-parser", "inkling", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:inkling-2-small thinkingmachines/Inkling-2-Small \\\n --tensor-parallel-size 4 \\\n --max-model-len 262144 \\\n --max-num-seqs 256 \\\n --reasoning-parser inkling \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:inkling-2-small", "thinkingmachines/Inkling-2-Small", "--tensor-parallel-size", "4", "--max-model-len", "262144", "--max-num-seqs", "256", "--reasoning-parser", "inkling", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "blackwell", "compute_arch": "sm100", "display_name": "B200", "description": "NVIDIA B200 SXM 180 GB HBM3e · 8-GPU HGX B200 node", "gpu_count": 8, "vram_gb": 1440, "multi_node": false }, "alternatives": { "multi_node_tp": "/thinkingmachines/Inkling-2-Small/strategies/multi_node_tp.json", "kv_store_centralized_mooncake": "/thinkingmachines/Inkling-2-Small/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/thinkingmachines/Inkling-2-Small/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "b200": "/thinkingmachines/Inkling-2-Small/hw/b200.json", "gb200": "/thinkingmachines/Inkling-2-Small/hw/gb200.json", "b300": "/thinkingmachines/Inkling-2-Small/hw/b300.json", "gb300": "/thinkingmachines/Inkling-2-Small/hw/gb300.json" } }, "model": { "model_id": "thinkingmachines/Inkling-2-Small", "min_vllm_version": "nightly", "nightly_required": true, "docker_image": "vllm/vllm-openai:inkling-2-small", "architecture": "moe", "parameter_count": "594B", "active_parameters": "17B", "context_length": 1048576, "base_args": [], "base_env": {}, "install": { "docker": { "command": "docker pull vllm/vllm-openai:inkling-2-small", "note": "Serve from the `vllm/vllm-openai:inkling-2-small` image; no pip wheel is available yet." } } }, "features": { "reasoning": { "description": "Parse reasoning output with the Inkling reasoning parser.", "args": [ "--reasoning-parser", "inkling" ] }, "tool_calling": { "description": "Enable auto tool choice with the Inkling tool-call parser.", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "inkling" ] }, "spec_decoding": { "description": "DSpark speculative decoding with the draft bundled in the checkpoint: a 5-layer parallel block drafter plus a recurrent head, 7 draft tokens per step. Drafts are sampled from the draft distribution, as in the reference, which raises acceptance at temperature > 0.", "args": [ "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}" ] } }, "opt_in_features": [ "tool_calling" ], "variants": { "default": { "precision": "nvfp4", "vram_minimum_gb": 473, "description": "Routed experts in NVFP4 (layers 0-39) and MXFP8 (layers 40-47) with activations quantized to the same format; attention, routers, attention-residual queries and the LM head in BF16/FP32. ~393 GB on disk including the 3.2 GB DSpark draft." } }, "hardware_overrides": {}, "troubleshooting": [ { "title": "To serve at least one request with the model's max seq len (1048576), (64.47 GiB KV cache is needed, which is larger than the available KV cache memory", "when": { "strategies": [ "single_node_tp" ] }, "body": "A 1M-token request does not fit on one 4-GPU tray. Keep `--max-model-len 262144` at TP4, or use the multi-node TP strategy with 2 nodes (TP8) for the full 1M context." }, { "title": "AssertionError: Executable was changed during NUMA binding context", "when": { "strategies": [ "multi_node_tp" ] }, "body": "On the headless node of a multi-node launch, vLLM's default NUMA binding swaps the global multiprocessing executable while workers start from concurrent threads. Launch every node with `--no-numa-bind`." } ], "guide": "## Overview\n\nInkling 2 Small is the next generation of Thinking Machines' [Inkling](https://huggingface.co/thinkingmachines/Inkling)\nand Inkling-Small: a multimodal Mixture-of-Experts model (text and image in, text out) with\n594B total and 17B active parameters and up to 1M tokens of context. It ships with a DSpark\nspeculator in the same checkpoint.\n\nWhat changed from Inkling:\n\n- **48 layers** (40 sliding-window + 8 full attention, the same 5:1 pattern) with\n relative-bias attention instead of RoPE and four short convolutions per layer, as in Inkling.\n- **Block attention residuals** replace the residual stream: before each sublayer, a learned\n query mixes the token embedding and the outputs of earlier 6-layer blocks.\n- **Gated attention output** with a per-token sink.\n- **384 routed experts, top-8, no shared experts.** Experts are NVFP4 in layers 0-39 and MXFP8\n in layers 40-47.\n- **Image encoder:** a 23-layer ViT. No audio input.\n- **DSpark** speculative decoding (7 tokens per step) instead of 8 chained MTP heads. Its\n recurrent Markov head is the RNN variant from the DSpark paper.\n\nInkling 2 Small runs its MoE expert-parallel across the tensor-parallel group, so TP4 means\nTP4 attention with EP4 experts, and TP8 means TP8 attention with EP8 experts.\n\n## Runtime\n\nUse the `vllm/vllm-openai:inkling-2-small` image until Inkling 2 Small support is in a vLLM release.\n\n## Configurations\n\nVerified on GB200 with DSpark and the reasoning parser enabled:\n\n- **One tray, TP4:** contexts up to 262,144 tokens.\n- **Two trays, TP8:** the full 1,048,576-token context.\n\nLoRA adapters are supported on the attention projections and the routed experts. Expert\nadapters need BF16 expert weights; with the NVFP4/MXFP8 experts of this checkpoint, use\nattention-only adapters.\n\nTool calling uses the `inkling` tool-call parser (opt-in above). Its parser is unit-tested;\nno end-to-end tool-calling benchmark has been run yet.\n\n## Client Usage\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\n\nresponse = client.chat.completions.create(\n model=\"thinkingmachines/Inkling-2-Small\",\n messages=[{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"text\", \"text\": \"Describe this image.\"},\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://example.com/photo.jpg\"}},\n ],\n }],\n)\nprint(response.choices[0].message.content)\n```\n\n## Benchmarking\n\nBenchmark through the chat endpoint so the server applies the chat template and the\nreasoning parser. DSpark acceptance depends on the prompts and the sampling temperature:\nrandom-token prompts accept almost nothing, so benchmark on realistic text such as SPEED-Bench.\nThe DSpark arguments above sample drafts probabilistically; with vLLM's default greedy\ndrafting, acceptance at temperature 1.0 is about 15-25% lower.\n\n```bash\nvllm bench serve \\\n --backend openai-chat --endpoint /v1/chat/completions \\\n --model thinkingmachines/Inkling-2-Small \\\n --dataset-name speed_bench --speed-bench-dataset-subset throughput_1k \\\n --speed-bench-output-len 1024 --max-concurrency 16 --num-prompts 64\n```\n\n## References\n\n- [Inkling 2 Small announcement](https://thinkingmachines.ai/news/inkling-2-small/)\n- [Inkling 2 Small on Hugging Face](https://huggingface.co/thinkingmachines/Inkling-2-Small)\n- [vLLM blog: Inkling 2 Small on vLLM](https://vllm.ai/blog/2026-11-12-inkling2-small)\n- [Inkling recipe](https://recipes.vllm.ai/thinkingmachines/Inkling)\n- [DeepSeek DSpark (DeepSpec)](https://github.com/deepseek-ai/DeepSpec)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "6363872519d46fa1767a8d38c5fe6f08a83061b2", "path": "recipes/models/thinkingmachines/Inkling-2-Small.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/thinkingmachines/Inkling-2-Small.yaml", "overlays": [] } }