{
  "hf_id": "thinkingmachines/Inkling-2-Small-MXFP8",
  "meta": {
    "title": "Inkling 2 Small MXFP8",
    "slug": "inkling-2-small-mxfp8",
    "provider": "Thinking Machines Lab",
    "description": "MXFP8 checkpoint of Inkling 2 Small, the multimodal (text and image in, text out) MoE from Thinking Machines Lab with 594B total and 17B active parameters, up to 1M context, and a bundled DSpark speculator.",
    "date_added": "2026-11-20",
    "date_updated": "2026-11-20",
    "tasks": [
      "multimodal"
    ],
    "related_recipes": [
      "thinkingmachines/Inkling-2-Small"
    ],
    "hardware": {
      "h200": "verified"
    }
  },
  "recommended_command": {
    "hardware": "h200",
    "strategy": "single_node_tp",
    "variant": "default",
    "node_count": 1,
    "deploy_type": "single_node",
    "env": {},
    "docker_image": "vllm/vllm-openai:nightly",
    "command": "vllm serve thinkingmachines/Inkling-2-Small-MXFP8 \\\n  --tensor-parallel-size 8 \\\n  --max-model-len 524288 \\\n  --max-num-seqs 256 \\\n  --reasoning-parser inkling \\\n  --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}'",
    "argv": [
      "vllm",
      "serve",
      "thinkingmachines/Inkling-2-Small-MXFP8",
      "--tensor-parallel-size",
      "8",
      "--max-model-len",
      "524288",
      "--max-num-seqs",
      "256",
      "--reasoning-parser",
      "inkling",
      "--speculative-config",
      "{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}"
    ],
    "docker_command": "docker run --gpus all \\\n  --privileged --ipc=host -p 8000:8000 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  vllm/vllm-openai:nightly thinkingmachines/Inkling-2-Small-MXFP8 \\\n  --tensor-parallel-size 8 \\\n  --max-model-len 524288 \\\n  --max-num-seqs 256 \\\n  --reasoning-parser inkling \\\n  --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}'",
    "docker_argv": [
      "docker",
      "run",
      "--gpus",
      "all",
      "--privileged",
      "--ipc=host",
      "-p",
      "8000:8000",
      "-v",
      "~/.cache/huggingface:/root/.cache/huggingface",
      "vllm/vllm-openai:nightly",
      "thinkingmachines/Inkling-2-Small-MXFP8",
      "--tensor-parallel-size",
      "8",
      "--max-model-len",
      "524288",
      "--max-num-seqs",
      "256",
      "--reasoning-parser",
      "inkling",
      "--speculative-config",
      "{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}"
    ],
    "strategy_spec": {
      "name": "single_node_tp",
      "deploy_type": "single_node",
      "display_name": "Tensor Parallel",
      "short_label": "TP",
      "orientation": "latency",
      "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n",
      "hardware_match": {
        "min_gpus": 1,
        "max_gpus": 8,
        "multi_node": false
      },
      "vllm_args": [],
      "parallel_flag": "--tensor-parallel-size"
    },
    "hardware_profile": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H200",
      "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node",
      "gpu_count": 8,
      "vram_gb": 1128,
      "multi_node": false
    },
    "alternatives": {
      "kv_store_centralized_mooncake": "/thinkingmachines/Inkling-2-Small-MXFP8/strategies/kv_store_centralized_mooncake.json",
      "kv_store_distributed_mooncake": "/thinkingmachines/Inkling-2-Small-MXFP8/strategies/kv_store_distributed_mooncake.json"
    },
    "by_hardware": {
      "h200": "/thinkingmachines/Inkling-2-Small-MXFP8/hw/h200.json",
      "b200": "/thinkingmachines/Inkling-2-Small-MXFP8/hw/b200.json",
      "gb200": "/thinkingmachines/Inkling-2-Small-MXFP8/hw/gb200.json",
      "b300": "/thinkingmachines/Inkling-2-Small-MXFP8/hw/b300.json",
      "gb300": "/thinkingmachines/Inkling-2-Small-MXFP8/hw/gb300.json"
    }
  },
  "model": {
    "model_id": "thinkingmachines/Inkling-2-Small-MXFP8",
    "min_vllm_version": "0.35.0",
    "nightly_required": true,
    "docker_image": "vllm/vllm-openai:nightly",
    "architecture": "moe",
    "parameter_count": "594B",
    "active_parameters": "17B",
    "context_length": 1048576,
    "base_args": [],
    "base_env": {},
    "install": {
      "pip": {
        "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --pre \\\n  --extra-index-url https://wheels.vllm.ai/nightly/cu130 \\\n  --extra-index-url https://download.pytorch.org/whl/cu130 \\\n  --index-strategy unsafe-best-match",
        "note": "vLLM 0.35.0 isn't released yet — nightly required. Nightly wheels are CUDA 13 (cu130) only; the host needs an r580+ NVIDIA driver."
      },
      "docker": {
        "command": "docker pull vllm/vllm-openai:nightly"
      }
    }
  },
  "features": {
    "reasoning": {
      "description": "Parse reasoning output with the Inkling reasoning parser.",
      "args": [
        "--reasoning-parser",
        "inkling"
      ]
    },
    "tool_calling": {
      "description": "Enable auto tool choice with the Inkling tool-call parser.",
      "args": [
        "--enable-auto-tool-choice",
        "--tool-call-parser",
        "inkling"
      ]
    },
    "spec_decoding": {
      "description": "DSpark speculative decoding with the draft bundled in the checkpoint: a 5-layer parallel block drafter plus a recurrent head, 7 draft tokens per step. Drafts are sampled from the draft distribution, as in the reference, which raises acceptance at temperature > 0.",
      "args": [
        "--speculative-config",
        "{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}"
      ]
    }
  },
  "opt_in_features": [
    "tool_calling"
  ],
  "variants": {
    "default": {
      "precision": "mxfp8",
      "precision_hardware": {
        "brand": "NVIDIA"
      },
      "vram_minimum_gb": 747,
      "supported_hardware": [
        "h200",
        "arch:sm100",
        "arch:sm103"
      ],
      "description": "All routed experts in MXFP8; attention, routers, attention-residual queries and the LM head in BF16/FP32. ~623 GB on disk including the DSpark draft. Blackwell quantizes expert activations to MXFP8; Hopper keeps them in BF16 (W8A16)."
    }
  },
  "strategy_hardware": {
    "multi_node_tp": {
      "hopper": "unsupported"
    }
  },
  "strategy_min_gpus": {
    "single_node_tp": 8,
    "multi_node_tp": 8
  },
  "hardware_overrides": {},
  "troubleshooting": [
    {
      "title": "AssertionError: Executable was changed during NUMA binding context",
      "when": {
        "strategies": [
          "multi_node_tp"
        ]
      },
      "body": "On the headless node of a multi-node launch, vLLM's default NUMA binding swaps the global multiprocessing executable while workers start from concurrent threads. Launch every node with `--no-numa-bind`."
    }
  ],
  "guide": "## Overview\n\nThis is the MXFP8 checkpoint of [Inkling 2 Small](https://recipes.vllm.ai/thinkingmachines/Inkling-2-Small):\nthe same model, with every routed expert in MXFP8 instead of NVFP4 (layers 0-39) and MXFP8\n(layers 40-47). Inkling 2 Small was trained in NVFP4, and Thinking Machines recommends the\nNVFP4 checkpoint for all use cases. MXFP8 scores 1-2 points higher on evals, at about 1.6x the\nmemory. See the NVFP4 recipe for the architecture, LoRA support, client usage and benchmarking.\n\n## Runtime\n\nInkling 2 Small is supported in vLLM main. Use the `vllm/vllm-openai:nightly` image or a\nnightly wheel until it is in a vLLM release.\n\n## Configurations\n\nThe checkpoint needs about 750 GB, so it runs at TP8 (TP8 attention with EP8 experts): one\n8-GPU H200, B200 or B300 node, or two GB200 or GB300 trays.\n\n- **H200, TP8:** contexts up to 524,288 tokens. With the command above (DSpark and the\n  reasoning parser), GSM8K is 0.972-0.974 (0-shot, chat) with a DSpark acceptance length of 4.5.\n- **Blackwell, TP8:** the full 1,048,576-token context.\n\nOn Hopper, the experts run as W8A16 GEMMs with BF16 activations.\n\n## References\n\n- [Inkling 2 Small recipe (NVFP4)](https://recipes.vllm.ai/thinkingmachines/Inkling-2-Small)\n- [Inkling 2 Small MXFP8 on Hugging Face](https://huggingface.co/thinkingmachines/Inkling-2-Small-MXFP8)\n- [Inkling 2 Small announcement](https://thinkingmachines.ai/news/inkling-2-small/)\n",
  "source": {
    "source": "vllm",
    "repo": "vllm-project/vllm",
    "ref": "main",
    "sha": "d53154d9f354ebd4a4deb4fd81935db0e266c5fc",
    "path": "recipes/models/thinkingmachines/Inkling-2-Small-MXFP8.yaml",
    "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/thinkingmachines/Inkling-2-Small-MXFP8.yaml",
    "overlays": []
  }
}