{ "hf_id": "MiniMaxAI/MiniMax-M3", "meta": { "title": "MiniMax-M3", "slug": "minimax-m3", "provider": "MiniMax", "description": "MiniMax M3 vision-language MoE (427B total / 26B active) for frontier coding, agent toolchains, and 1M-token reasoning via MSA sparse attention — native multimodal (image + video + computer use); BF16 plus MXFP8, NVIDIA Blackwell NVFP4, and AMD MI355X MXFP4 variants. Runs on NVIDIA (Hopper/Blackwell) and AMD CDNA4/CDNA3.", "date_added": "2026-06-12", "date_updated": "2026-10-26", "tasks": [ "text", "multimodal" ], "performance_headline": "Frontier coding and agent (SWE-Bench Pro 59.0, Terminal-Bench 2.1 66.0); MSA sparse attention; 1M context", "related_recipes": [], "hardware": { "h200": "verified", "b200": "verified", "b300": "verified", "gb200": "verified", "mi300x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:minimax-m3", "command": "vllm serve MiniMaxAI/MiniMax-M3 \\\n --block-size 128 \\\n --max-model-len 65536 \\\n --tensor-parallel-size 8 \\\n --tool-call-parser minimax_m3 \\\n --enable-auto-tool-choice \\\n --reasoning-parser minimax_m3", "argv": [ "vllm", "serve", "MiniMaxAI/MiniMax-M3", "--block-size", "128", "--max-model-len", "65536", "--tensor-parallel-size", "8", "--tool-call-parser", "minimax_m3", "--enable-auto-tool-choice", "--reasoning-parser", "minimax_m3" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:minimax-m3 MiniMaxAI/MiniMax-M3 \\\n --block-size 128 \\\n --max-model-len 65536 \\\n --tensor-parallel-size 8 \\\n --tool-call-parser minimax_m3 \\\n --enable-auto-tool-choice \\\n --reasoning-parser minimax_m3", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:minimax-m3", "MiniMaxAI/MiniMax-M3", "--block-size", "128", "--max-model-len", "65536", "--tensor-parallel-size", "8", "--tool-call-parser", "minimax_m3", "--enable-auto-tool-choice", "--reasoning-parser", "minimax_m3" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/MiniMaxAI/MiniMax-M3/strategies/single_node_tep.json", "single_node_dep": "/MiniMaxAI/MiniMax-M3/strategies/single_node_dep.json", "multi_node_tp": "/MiniMaxAI/MiniMax-M3/strategies/multi_node_tp.json", "multi_node_tp_pp": "/MiniMaxAI/MiniMax-M3/strategies/multi_node_tp_pp.json", "multi_node_tep": "/MiniMaxAI/MiniMax-M3/strategies/multi_node_tep.json", "multi_node_dep": "/MiniMaxAI/MiniMax-M3/strategies/multi_node_dep.json", "pd_cluster": "/MiniMaxAI/MiniMax-M3/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/MiniMaxAI/MiniMax-M3/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/MiniMaxAI/MiniMax-M3/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/MiniMaxAI/MiniMax-M3/hw/h200.json", "h100": "/MiniMaxAI/MiniMax-M3/hw/h100.json", "b200": "/MiniMaxAI/MiniMax-M3/hw/b200.json", "gb200": "/MiniMaxAI/MiniMax-M3/hw/gb200.json", "b300": "/MiniMaxAI/MiniMax-M3/hw/b300.json", "gb300": "/MiniMaxAI/MiniMax-M3/hw/gb300.json", "mi300x": "/MiniMaxAI/MiniMax-M3/hw/mi300x.json", "mi325x": "/MiniMaxAI/MiniMax-M3/hw/mi325x.json", "mi350x": "/MiniMaxAI/MiniMax-M3/hw/mi350x.json", "mi355x": "/MiniMaxAI/MiniMax-M3/hw/mi355x.json" } }, "model": { "model_id": "MiniMaxAI/MiniMax-M3", "smg_grpc": { "reasoning_parser": "minimax_m3", "tool_call_parser": "minimax_m3" }, "min_vllm_version": "0.24.0", "nightly_required": true, "docker_image": { "nvidia": "vllm/vllm-openai:minimax-m3", "amd": "vllm/vllm-openai-rocm:nightly" }, "architecture": "moe", "parameter_count": "427B", "active_parameters": "26B", "context_length": 1048576, "base_args": [ "--block-size", "128" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --pre \\\n --extra-index-url https://wheels.vllm.ai/nightly/cu130 \\\n --extra-index-url https://download.pytorch.org/whl/cu130 \\\n --index-strategy unsafe-best-match", "note": "vLLM 0.24.0 isn't released yet — nightly required. Nightly wheels are CUDA 13 (cu130) only; the host needs an r580+ NVIDIA driver." }, "docker": { "command": "docker pull vllm/vllm-openai:minimax-m3" } } }, "features": { "tool_calling": { "description": "MiniMax M3 tool call parser with automatic tool choice", "args": [ "--tool-call-parser", "minimax_m3", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "MiniMax M3 reasoning parser (supports `thinking` and `non-thinking` modes, switchable per request)", "args": [ "--reasoning-parser", "minimax_m3" ] }, "spec_decoding": { "description": "Eagle3 speculative decoding with an Inferact MiniMax-M3 draft head — accelerates decoding. Choose between the standard MHA head and the GQA head (16× smaller draft KV cache — better for long-context or high-batch deployments).", "default_mode": "eagle3", "modes": { "eagle3": { "label": "MHA", "description": "Standard Eagle3 draft head (Multi-Head Attention, 64 KV heads).", "args": [ "--speculative-config", "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" ], "hardware_overrides": { "amd": { "args": [ "--speculative-config", "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"TRITON_ATTN\"}" ] } } }, "eagle3_gqa": { "label": "GQA", "description": "GQA Eagle3 draft head (Grouped Query Attention, 4 KV heads vs 64) — 16× smaller draft KV cache. Prefer for long-context or large-batch deployments where draft KV memory is the constraint.", "args": [ "--speculative-config", "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3-GQA\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}" ], "hardware_overrides": { "gb200": { "args": [ "--speculative-config", "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3-GQA\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASHINFER\", \"use_local_argmax_reduction\": true}" ] }, "b200": { "args": [ "--speculative-config", "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3-GQA\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASHINFER\", \"use_local_argmax_reduction\": true}" ] }, "b300": { "args": [ "--speculative-config", "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3-GQA\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASHINFER\", \"use_local_argmax_reduction\": true}" ] }, "amd": { "args": [ "--speculative-config", "{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3-GQA\", \"num_speculative_tokens\": 3, \"attention_backend\": \"TRITON_ATTN\"}" ] } } } } }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "thinking_always_on": { "description": "Pin `thinking_mode: enabled` server-side — every turn reasons before answering, including after tool results. Recommended for agentic workflows.", "args": [ "--default-chat-template-kwargs", "{\"thinking_mode\": \"enabled\"}" ] }, "encoder_parallel": { "description": "Run the vision encoder data-parallel instead of tensor-parallel — avoids TP comm overhead on the small encoder. Also enables the host-shared-memory multimodal processor cache and the per-brand encoder attention backend (FlashInfer on NVIDIA, AITER FlashAttention on AMD). Mutually exclusive with text_only.", "hardware_overrides": { "nvidia": { "args": [ "--mm-encoder-attn-backend", "FLASHINFER", "--mm-processor-cache-type", "shm", "--mm-encoder-tp-mode", "data" ] }, "amd": { "args": [ "--mm-encoder-attn-backend", "ROCM_AITER_FA", "--mm-processor-cache-type", "shm", "--mm-encoder-tp-mode", "data" ] } } } }, "opt_in_features": [ "spec_decoding", "text_only", "thinking_always_on", "encoder_parallel" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 1025, "description": "Native BF16 checkpoint — multi-node TP recommended for long-context headroom", "hardware_overrides": { "h200": { "extra_args": [ "--max-model-len", "65536" ] } } }, "mxfp8": { "model_id": "MiniMaxAI/MiniMax-M3-MXFP8", "precision": "mxfp8", "vram_minimum_gb": 513, "description": "NVIDIA-quantized MXFP8 weights — Blackwell (B200/B300) for native MX tensor cores, and AMD CDNA4 (MI350X/MI355X, gfx950) for native MXFP8 Matrix Cores.", "extra_args": [ "--attention_config.indexer_kv_dtype", "fp8" ], "hardware_overrides": { "blackwell": { "extra_env": { "VLLM_FLOAT32_MATMUL_PRECISION": "high" } }, "h100": { "extra_args": [ "--max-model-len", "131072" ] }, "h200": { "docker_image": "vllm/vllm-openai:nightly-1a001d584244bd21829a5e5bd4a376ea141c0ce3", "extra_args": [ "--attention-backend", "FLASH_ATTN", "--linear-backend", "humming", "--kv-cache-dtype", "fp8", "--gpu-memory-utilization", "0.95", "--enable-prefix-caching" ], "extra_env": { "VLLM_MARLIN_USE_ATOMIC_ADD": "1" } }, "mi300x": { "docker_image": "vllm/vllm-openai-rocm:nightly", "extra_args": [ "--language-model-only", "--async-scheduling", "--max-num-batched-tokens", "16384" ], "extra_env": { "VLLM_ENGINE_READY_TIMEOUT_S": "3600", "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "0", "TORCH_BLAS_PREFER_HIPBLASLT": "1", "NCCL_MIN_NCHANNELS": "112", "GPU_MAX_HW_QUEUES": "2" } }, "mi355x": { "docker_image": "vllm/vllm-openai-rocm:nightly", "extra_args": [ "--linear-backend", "emulation" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "json": "/MiniMaxAI/MiniMax-M3-MXFP8.json" }, "nvfp4": { "model_id": "nvidia/MiniMax-M3-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 257, "description": "NVIDIA-quantized NVFP4 weights (ModelOpt) for Blackwell (B200/B300/GB200) — native FP4 tensor cores, ~half the VRAM of MXFP8.", "extra_args": [ "--trust-remote-code", "--kv-cache-dtype", "fp8", "--attention_config.backend", "FLASHINFER", "--attention_config.use_trtllm_attention", "true", "--attention_config.indexer_kv_dtype", "fp8", "--attention_config.minimax_m3_msa_decode_backend", "cutlass" ], "extra_env": { "VLLM_FLOAT32_MATMUL_PRECISION": "high", "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" }, "hardware_overrides": { "b200": { "docker_image": "vllm/vllm-openai:nightly-af7f9488c2210d67e1033ecdc845b087ee7fe92b", "extra_args": [ "--enable-prefix-caching", "--max-num-batched-tokens", "16384", "--gpu-memory-utilization", "0.95", "--disable-flashinfer-autotune", "--stream-interval", "20" ] }, "b300": { "docker_image": "vllm/vllm-openai:nightly-49cb2bbf71cfd36ee5aa33d1e184b1cddff55294", "extra_args": [ "--enable-prefix-caching", "--max-num-batched-tokens", "16384", "--gpu-memory-utilization", "0.95", "--no-enable-flashinfer-autotune", "--stream-interval", "20" ] }, "gb200": { "docker_image": "vllm/vllm-openai:nightly-5555a2d8f7116b2b4877f66516f4128176cf722a", "extra_args": [ "--enable-prefix-caching", "--max-num-batched-tokens", "16384", "--gpu-memory-utilization", "0.9", "--no-enable-flashinfer-autotune", "--stream-interval", "20" ], "extra_env": { "NCCL_CUMEM_ENABLE": "1", "NCCL_MNNVL_ENABLE": "1", "NCCL_NVLS_ENABLE": "1" } } }, "json": "/nvidia/MiniMax-M3-NVFP4.json" }, "mxfp4": { "model_id": "amd/MiniMax-M3-MXFP4", "precision": "mxfp4", "vram_minimum_gb": 257, "description": "AMD Quark OCP MXFP4 weights and dynamic MXFP4 activations for MI355X (gfx950).", "supported_hardware": [ "mi355x" ], "tp": 8, "hardware_overrides": { "mi355x": { "docker_image": "vllm/vllm-openai-rocm:nightly", "extra_args": [ "--trust-remote-code", "--compilation-config", "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}", "--mm-encoder-tp-mode", "data", "--mm-encoder-attn-backend", "ROCM_AITER_FA" ] } }, "json": "/amd/MiniMax-M3-MXFP4.json" } }, "hardware_overrides": { "amd": { "extra_args": [ "--attention-backend", "TRITON_ATTN" ], "extra_env": { "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1" } } }, "presets": { "h200_agentx": { "label": "AgentX TP8", "hardware": "h200", "strategy": "single_node_tp", "variants": [ "mxfp8" ], "workload": "Agentic (InferenceX AgentX)", "description": "One 8-GPU H200 node, TP8, with the GQA EAGLE3 draft (three tokens) and thinking always on; the scheduler batch is twice the concurrency.", "source": { "label": "InferenceX #4429", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4429", "key": "minimaxm3-fp8-h200-vllm-agentic-mtp" }, "image": "vllm/vllm-openai:nightly-83c078d955cd6e0c63d3b12abe669c8de936a370@sha256:cb12c3016938a4ee98ed176ac833984089460689a0982cff4b5a60bb45b3cded", "select": { "features": { "spec_decoding": "eagle3_gqa", "text_only": true, "thinking_always_on": true } }, "args": [ "--trust-remote-code", "--block-size", "128" ], "points": { "c1": { "label": "1", "concurrency": 1, "args": [ "--max-num-seqs", "2", "--max-cudagraph-capture-size", "8" ] }, "c2": { "label": "2", "concurrency": 2, "args": [ "--max-num-seqs", "4", "--max-cudagraph-capture-size", "16" ] }, "c4": { "label": "4", "concurrency": 4, "args": [ "--max-num-seqs", "8", "--max-cudagraph-capture-size", "32" ] }, "c6": { "label": "6", "concurrency": 6, "args": [ "--max-num-seqs", "12", "--max-cudagraph-capture-size", "48" ] }, "c8": { "label": "8", "concurrency": 8, "args": [ "--max-num-seqs", "16", "--max-cudagraph-capture-size", "64" ] }, "c10": { "label": "10", "concurrency": 10, "args": [ "--max-num-seqs", "20", "--max-cudagraph-capture-size", "80" ] }, "c12": { "label": "12", "concurrency": 12, "args": [ "--max-num-seqs", "24", "--max-cudagraph-capture-size", "96" ] }, "c14": { "label": "14", "concurrency": 14, "args": [ "--max-num-seqs", "28", "--max-cudagraph-capture-size", "112" ] } }, "runbook": "The H200 AgentX lane in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) (`minimaxm3-fp8-h200-vllm-agentic-mtp`, [SemiAnalysisAI/InferenceX#4429](https://github.com/SemiAnalysisAI/InferenceX/pull/4429)) serves the MXFP8 checkpoint with the GQA EAGLE3 draft (three speculative tokens) as one aggregated vLLM worker on a single 8-GPU H200 node: TP8, no expert parallelism, no host KV offload.\n\nThe InferenceX throughput points replaced block verification with synthetic rejection at the committed golden EAGLE3-GQA acceptance length (2.78 for MiniMax-M3 with three draft tokens, thinking on; [`minimaxm3_eagle3_gqa.yaml`](https://github.com/SemiAnalysisAI/InferenceX/blob/main/inferencex-e2e/infx/golden_al_distribution/minimaxm3_eagle3_gqa.yaml)) so every point is measured at the same acceptance rate. The command above does not set it.\n", "json": "/MiniMaxAI/MiniMax-M3/presets/h200_agentx.json" }, "b200_agentx": { "label": "AgentX TP4", "hardware": "b200", "strategy": "single_node_tp", "variants": [ "nvfp4" ], "workload": "Agentic (InferenceX AgentX)", "description": "TP4 on four of the node's eight B200 GPUs with the GQA EAGLE3 draft (three tokens, local argmax reduction); the larger points add host KV offload.", "source": { "label": "InferenceX #4382", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4382", "key": "minimaxm3-fp4-b200-vllm-agentic-mtp" }, "image": "vllm/vllm-openai:nightly-af7f9488c2210d67e1033ecdc845b087ee7fe92b@sha256:8e2ffbb9f866de678b22e7446903b1ee5a5766c51a948013277477c462a68a77", "tp": 4, "select": { "features": { "spec_decoding": "eagle3_gqa", "text_only": true, "thinking_always_on": true } }, "points": { "c15": { "label": "15", "concurrency": 15, "args": [ "--max-num-seqs", "30", "--max-cudagraph-capture-size", "120" ], "remove_args": [ "--attention_config.minimax_m3_msa_decode_backend" ], "select": { "kv_offload": "off" } }, "c30": { "label": "30 (KV offload)", "concurrency": 30, "args": [ "--max-num-seqs", "60", "--max-cudagraph-capture-size", "4096" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": null, "cpu_bytes_to_use": 1073000000000, "lazy_offload": true } }, "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 256, 320, 384, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096 ] } } }, "c32": { "label": "32 (KV offload)", "concurrency": 32, "args": [ "--max-num-seqs", "64", "--max-cudagraph-capture-size", "256" ], "remove_args": [ "--attention_config.minimax_m3_msa_decode_backend" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": null, "cpu_bytes_to_use": 1073000000000, "lazy_offload": true } } } }, "c38": { "label": "38 (KV offload)", "concurrency": 38, "args": [ "--max-num-seqs", "76", "--max-cudagraph-capture-size", "4096" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": null, "cpu_bytes_to_use": 1073000000000, "lazy_offload": true } }, "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 384, 448, 512, 640, 768, 896, 1024, 1280, 1536, 1792, 2048, 2560, 3072, 3584, 4096 ] } } } }, "runbook": "The B200 AgentX lane in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) (`minimaxm3-fp4-b200-vllm-agentic-mtp` and `minimaxm3-fp4-b200-vllm-agentic-mtp-c10`, [SemiAnalysisAI/InferenceX#4382](https://github.com/SemiAnalysisAI/InferenceX/pull/4382)) serves the NVFP4 checkpoint with the GQA EAGLE3 draft as one aggregated vLLM worker on a single 8-GPU B200 node. Most points run TP4; the concurrency-10 arm also has a TP8 point.\n\nThe host-KV-offload points pin 1,073 GB of host memory for KV blocks across the node (lazy offload).\n\nThe **Verified** badge for this GPU is recipe-wide. This preset covers the NVFP4 variant only; the GQA draft override in `features.spec_decoding` also applies to the other variants on this GPU, which were not qualified.\n\nThe InferenceX throughput points replaced block verification with synthetic rejection at the committed golden EAGLE3-GQA acceptance length so every point is measured at the same acceptance rate. Production serving keeps the default rejection sampler, as in the command above.\n", "json": "/MiniMaxAI/MiniMax-M3/presets/b200_agentx.json" }, "b200_agentx_c10": { "label": "AgentX active-seq caps", "hardware": "b200", "strategy": "single_node_tp", "variants": [ "nvfp4" ], "workload": "Agentic (InferenceX AgentX)", "description": "Caps the sequences the scheduler keeps active (--max-num-active-seqs) well below the concurrency, with a five- or three-token GQA draft without local argmax reduction.", "source": { "label": "InferenceX #4382", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4382", "key": "minimaxm3-fp4-b200-vllm-agentic-mtp-c10" }, "image": "vllm/vllm-openai:nightly-ac7509e2b1db40fec2f03dde1ed4e9dfdc2338c9@sha256:98adb2311118263a453ec0acd5d4b72bda3db26cecc0fe8ffc94b9ab03c951f9", "tp": 4, "select": { "features": { "spec_decoding": "eagle3_gqa", "text_only": true, "thinking_always_on": true } }, "env": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "mnnvl" }, "json_args": { "--speculative-config": { "use_local_argmax_reduction": null } }, "points": { "c10_tp4": { "label": "10 (TP4)", "concurrency": 10, "args": [ "--max-num-active-seqs", "1", "--gpu-memory-utilization", "0.925", "--max-cudagraph-capture-size", "4096" ], "select": { "kv_offload": "off" }, "json_args": { "--speculative-config": { "num_speculative_tokens": 5 }, "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096 ] } } }, "c10_tp8": { "label": "10 (TP8)", "concurrency": 10, "tp": 8, "args": [ "--max-num-active-seqs", "1", "--gpu-memory-utilization", "0.9", "--max-cudagraph-capture-size", "4096" ], "remove_args": [ "--disable-flashinfer-autotune" ], "select": { "kv_offload": "off" }, "json_args": { "--speculative-config": { "num_speculative_tokens": 5 }, "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096 ] } } }, "c20_a4": { "label": "20 (4 active)", "concurrency": 20, "args": [ "--max-num-active-seqs", "4", "--gpu-memory-utilization", "0.9", "--max-cudagraph-capture-size", "8192" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--speculative-config": { "num_speculative_tokens": 3 }, "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192 ] }, "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": null, "cpu_bytes_to_use": 1073000000000, "lazy_offload": true } } } }, "c20_a3": { "label": "20 (3 active)", "concurrency": 20, "args": [ "--max-num-active-seqs", "3", "--gpu-memory-utilization", "0.9", "--max-cudagraph-capture-size", "8192" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--speculative-config": { "num_speculative_tokens": 3 }, "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096, 6144, 8192 ] }, "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": null, "cpu_bytes_to_use": 1073000000000, "lazy_offload": true } } } }, "c32_a6": { "label": "32 (6 active)", "concurrency": 32, "args": [ "--max-num-active-seqs", "6", "--gpu-memory-utilization", "0.925", "--max-cudagraph-capture-size", "4096" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--speculative-config": { "num_speculative_tokens": 3 }, "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096 ] }, "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": null, "cpu_bytes_to_use": 1073000000000, "lazy_offload": true } } } }, "c32_a16": { "label": "32 (16 active)", "concurrency": 32, "args": [ "--max-num-active-seqs", "16", "--gpu-memory-utilization", "0.925", "--max-cudagraph-capture-size", "4096" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--speculative-config": { "num_speculative_tokens": 3 }, "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204, 208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 248, 252, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512, 640, 768, 1024, 1536, 2048, 3072, 4096 ] }, "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": null, "cpu_bytes_to_use": 1073000000000, "lazy_offload": true } } } } }, "runbook": "The B200 AgentX lane in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) (`minimaxm3-fp4-b200-vllm-agentic-mtp` and `minimaxm3-fp4-b200-vllm-agentic-mtp-c10`, [SemiAnalysisAI/InferenceX#4382](https://github.com/SemiAnalysisAI/InferenceX/pull/4382)) serves the NVFP4 checkpoint with the GQA EAGLE3 draft as one aggregated vLLM worker on a single 8-GPU B200 node. Most points run TP4; the concurrency-10 arm also has a TP8 point.\n\nThe **Verified** badge for this GPU is recipe-wide. This preset covers the NVFP4 variant only; the GQA draft override in `features.spec_decoding` also applies to the other variants on this GPU, which were not qualified.\n\nThe InferenceX throughput points replaced block verification with synthetic rejection at the committed golden EAGLE3-GQA acceptance length so every point is measured at the same acceptance rate. Production serving keeps the default rejection sampler, as in the command above.\n", "json": "/MiniMaxAI/MiniMax-M3/presets/b200_agentx_c10.json" }, "b300_agentx": { "label": "AgentX", "hardware": "b300", "strategy": "single_node_tp", "variants": [ "nvfp4" ], "workload": "Agentic (InferenceX AgentX)", "description": "One B300 node: TP8 at concurrency 1, TP4 at 20 and 24, TP2 with host KV offload at 24 and 28; GQA EAGLE3 draft (three tokens, local argmax reduction).", "source": { "label": "InferenceX #4357", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4357", "key": "minimaxm3-fp4-b300-vllm-agentic-mtp" }, "image": "vllm/vllm-openai:nightly-49cb2bbf71cfd36ee5aa33d1e184b1cddff55294@sha256:8c1434bfee7d7626eef78ef70600434ffba4a1b8813cb1d1c8707ad25f47035e", "select": { "features": { "spec_decoding": "eagle3_gqa", "text_only": true, "thinking_always_on": true } }, "json_args": { "--compilation-config": { "cudagraph_mode": "FULL_AND_PIECEWISE" } }, "points": { "c1_tp8": { "label": "1 (TP8)", "concurrency": 1, "tp": 8, "args": [ "--max-num-seqs", "2", "--gpu-memory-utilization", "0.95" ], "remove_args": [ "--attention_config.minimax_m3_msa_decode_backend" ], "env": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" }, "select": { "kv_offload": "off" }, "json_args": { "--compilation-config": { "cudagraph_capture_sizes": [ 4, 8, 256, 512, 1024, 2048 ] } } }, "c20_tp4": { "label": "20 (TP4)", "concurrency": 20, "tp": 4, "args": [ "--max-num-seqs", "30", "--max-num-active-seqs", "6", "--gpu-memory-utilization", "0.95" ], "remove_args": [ "--attention_config.minimax_m3_msa_decode_backend" ], "env": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" }, "select": { "kv_offload": "off" }, "json_args": { "--compilation-config": { "cudagraph_capture_sizes": [ 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 256, 512, 1024, 2048, 3072, 4096, 5120, 6144, 7168, 8192, 10240, 12288, 14336, 16384 ] } } }, "c24_tp4": { "label": "24 (TP4)", "concurrency": 24, "tp": 4, "args": [ "--max-num-seqs", "7", "--gpu-memory-utilization", "0.95" ], "remove_args": [ "--attention_config.minimax_m3_msa_decode_backend" ], "env": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "mnnvl" }, "select": { "kv_offload": "off" }, "json_args": { "--compilation-config": { "cudagraph_capture_sizes": [ 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 256, 512, 1024, 2048, 2560, 3072, 3584, 4096, 5120, 6144, 7168, 8192 ] } } }, "c24_tp2": { "label": "24 (TP2, KV offload)", "concurrency": 24, "tp": 2, "args": [ "--max-num-seqs", "36", "--gpu-memory-utilization", "0.92" ], "env": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm", "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "select": { "kv_offload": "simple" }, "json_args": { "--compilation-config": { "cudagraph_capture_sizes": [ 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 256, 512, 1024, 2048 ] }, "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": null, "cpu_bytes_to_use": 749000000000, "lazy_offload": true } } } }, "c28_tp2": { "label": "28 (TP2, KV offload)", "concurrency": 28, "tp": 2, "args": [ "--max-num-seqs", "36", "--gpu-memory-utilization", "0.92" ], "env": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm", "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "select": { "kv_offload": "simple" }, "json_args": { "--compilation-config": { "cudagraph_capture_sizes": [ 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 256, 512, 1024, 2048 ] }, "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": null, "cpu_bytes_to_use": 749000000000, "lazy_offload": true } } } } }, "runbook": "The B300 AgentX lane in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) (`minimaxm3-fp4-b300-vllm-agentic-mtp`, [SemiAnalysisAI/InferenceX#4357](https://github.com/SemiAnalysisAI/InferenceX/pull/4357)) serves the NVFP4 checkpoint with the GQA EAGLE3 draft (three speculative tokens) as one aggregated vLLM worker on a single B300 node: TP8 at concurrency 1, TP4 at concurrency 20 and 24, and TP2 at 24 and 28 with host KV offload.\n\nThe TP2 points pin 749 GB of host memory for KV blocks (lazy offload).\n\nThe **Verified** badge for this GPU is recipe-wide. This preset covers the NVFP4 variant only; the GQA draft override in `features.spec_decoding` also applies to the other variants on this GPU, which were not qualified.\n\nThe InferenceX throughput points replaced block verification with synthetic rejection at the committed golden EAGLE3-GQA acceptance length (2.78 for MiniMax-M3 with three draft tokens, thinking on; [`minimaxm3_eagle3_gqa.yaml`](https://github.com/SemiAnalysisAI/InferenceX/blob/main/inferencex-e2e/infx/golden_al_distribution/minimaxm3_eagle3_gqa.yaml)) so every point is measured at the same acceptance rate. The command above does not set it.\n", "json": "/MiniMaxAI/MiniMax-M3/presets/b300_agentx.json" }, "gb200_agentx": { "label": "AgentX TP4", "hardware": "gb200", "strategy": "single_node_tp", "variants": [ "nvfp4" ], "workload": "Agentic (InferenceX AgentX)", "description": "One TP4 worker per 4-GPU GB200 tray with the GQA EAGLE3 draft (three tokens, local argmax reduction), the CUTLASS MSA decode kernel and thinking always on, behind the Dynamo KV-aware router; the larger points add host KV offload.", "source": { "label": "InferenceX #4281", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4281", "key": "minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp", "revision": "bbebca8b4be98a06401e943bad37874abbd65101" }, "image": { "vllm": "vllm/vllm-openai:nightly-5555a2d8f7116b2b4877f66516f4128176cf722a@sha256:8896fd7b29fc4ee7ce839f146cb4757984c476ec79d0234ffa8c995dee6e339a", "pip": [ "ai-dynamo-runtime==1.6.0.dev20261016", "ai-dynamo==1.6.0.dev20261016" ] }, "tp": 4, "select": { "kv_offload": "off", "features": { "tool_calling": true, "reasoning": true, "spec_decoding": "eagle3_gqa", "text_only": true, "thinking_always_on": true, "encoder_parallel": false } }, "args": [ "--max-model-len", "1048576" ], "env": { "VLLM_ENGINE_READY_TIMEOUT_S": "7200", "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": "1800" }, "json_args": { "--compilation-config": { "cudagraph_mode": "FULL_AND_PIECEWISE" } }, "points": { "tp4_c1": { "label": "1", "concurrency": 1, "args": [ "--max-num-seqs", "2" ], "json_args": { "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 256, 512, 1024, 2048 ] } } }, "tp4_c10": { "label": "10", "concurrency": 10, "args": [ "--max-num-seqs", "16" ], "json_args": { "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 256, 512, 1024, 2048 ] } } }, "tp4_c20": { "label": "20 (KV offload)", "concurrency": 20, "args": [ "--max-num-seqs", "8" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use": 777389080576, "cpu_bytes_to_use_per_rank": 194347270144, "lazy_offload": true } }, "--compilation-config": { "cudagraph_capture_sizes": [ 4, 8, 12, 16, 20, 24, 28, 32, 256, 512, 1024, 2048 ] } } }, "tp4_c22": { "label": "22 (KV offload)", "concurrency": 22, "args": [ "--max-num-seqs", "12" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use": 777389080576, "cpu_bytes_to_use_per_rank": 194347270144, "lazy_offload": true } }, "--compilation-config": { "cudagraph_capture_sizes": [ 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 256, 512, 1024, 2048 ] } } }, "tp4_c25": { "label": "25 (KV offload)", "concurrency": 25, "args": [ "--max-num-seqs", "16" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use": 777389080576, "cpu_bytes_to_use_per_rank": 194347270144, "lazy_offload": true } }, "--compilation-config": { "cudagraph_capture_sizes": [ 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 256, 512, 1024, 2048 ] } } }, "tp4_c30": { "label": "30 (KV offload)", "concurrency": 30, "args": [ "--max-num-seqs", "45" ], "select": { "kv_offload": "simple" }, "env": { "VLLM_USE_SIMPLE_KV_OFFLOAD": "1" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use": 777389080576, "cpu_bytes_to_use_per_rank": 194347270144, "lazy_offload": true } }, "--compilation-config": { "cudagraph_capture_sizes": [ 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92, 96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152, 156, 160, 164, 168, 172, 176, 180, 256, 512, 1024, 2048 ] } } } }, "runbook": "The GB200 AgentX lane in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) (`minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp`, [SemiAnalysisAI/InferenceX#4281](https://github.com/SemiAnalysisAI/InferenceX/pull/4281)) serves the NVFP4 checkpoint with the GQA EAGLE3 draft (three speculative tokens, FlashInfer draft attention, local argmax reduction) and the CUTLASS MSA decode kernel as aggregated vLLM workers behind the Dynamo frontend's KV-aware router. This preset holds its TP4 points, one worker per 4-GPU tray; [AgentX TP8](#preset-gb200_agentx_tp8) holds the two-tray TP8 points.\n\n- **GPU-resident points** keep the whole KV cache in HBM. Their CUDA graph sizes start at 1 and 2 so the drafter's decode steps also replay full graphs.\n- **KV offload points** add a host-DRAM KV tier through `SimpleCPUOffloadConnector` with lazy offload: 181 GiB of pinned host memory per rank, 724 GiB per tray. Where `--max-num-seqs` is below the concurrency, requests beyond it wait in the scheduler queue.\n\n**Running it under Dynamo.** The lane ran Dynamo `1.6.0.dev20261016`, installed in the pinned image with `pip install --extra-index-url https://pypi.nvidia.com ai-dynamo-runtime==1.6.0.dev20261016 ai-dynamo==1.6.0.dev20261016`. Start etcd and NATS (JetStream on, 32 MB max payload) once:\n\n```bash\netcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://:2379\nprintf 'max_payload: 33554432\\njetstream {}\\n' > nats.conf && nats-server -c nats.conf\n```\n\nOn each worker tray, export `ETCD_ENDPOINTS=http://:2379` and `NATS_SERVER=nats://:4222` along with the generated environment, and run the generated flags with `python3 -m dynamo.vllm --model nvidia/MiniMax-M3-NVFP4` in place of `vllm serve nvidia/MiniMax-M3-NVFP4`. `dynamo.vllm` hands engine flags to vLLM's own parser, so the dotted `--attention_config.*` flags work as generated (the lane passed the same settings as one JSON `--attention-config`). It rejects vLLM's API-server flags, so leave out `--tool-call-parser minimax_m3 --enable-auto-tool-choice` and `--default-chat-template-kwargs '{\"thinking_mode\": \"enabled\"}'`, and add:\n\n```bash\n--dyn-tool-call-parser minimax_m3 --dyn-reasoning-parser minimax_m3 --request-plane tcp \\\n --kv-events-config '{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"enable_kv_cache_events\":true,\"endpoint\":\"tcp://*:5200\"}'\n```\n\nThe Dynamo frontend then parses tool calls and reasoning. To keep thinking on, send `\"thinking\": true` at the top level of each request, as the lane's client did, or add `--dyn-default-thinking-mode enabled` to the worker command; a request's own `thinking` or `chat_template_kwargs` still overrides that default.\n\nThen start the frontend:\n\n```bash\nexport ETCD_ENDPOINTS=http://:2379 NATS_SERVER=nats://:4222 DYN_REQUEST_PLANE=tcp\nexport DYN_TOKENIZER=fastokens DYN_TOKENIZER_CACHE_BYTES=8589934592 DYN_TCP_REQUEST_TIMEOUT=60\npython3 -m dynamo.frontend --trust-remote-code \\\n --router-mode kv --router-kv-events --router-temperature 0 \\\n --router-session-affinity-ttl-secs 14400 --kv-cache-block-size 128\n```\n\nThe KV router uses the workers' KV events to send each request to the worker holding its prefix, and session affinity keeps a conversation on that worker: send a stable conversation ID in the `X-Dynamo-Session-ID` request header. This Dynamo build warns that `--router-temperature` is deprecated and will be removed in v1.7; newer builds take `router_temperature` in the default policy parameters of `--router-policy-config`.\n\nThe **Verified** badge for this GPU is recipe-wide. These presets cover the NVFP4 variant only; the GQA draft and TP8 all-reduce overrides also apply to the other variants on GB200, which were not qualified.\n\nThe InferenceX throughput points replaced block verification with synthetic rejection at the committed golden EAGLE3-GQA acceptance length (2.78 for MiniMax-M3 with three draft tokens, thinking on; [`minimaxm3_eagle3_gqa.yaml`](https://github.com/SemiAnalysisAI/InferenceX/blob/main/inferencex-e2e/infx/golden_al_distribution/minimaxm3_eagle3_gqa.yaml)) so every point is measured at the same acceptance rate. Production serving keeps the default rejection sampler, as in the command above; the lane's accuracy evals ran that way. **Thinking Always On** pins `thinking_mode: enabled`, which the lane's client asked for on every request; without it the chat template's adaptive mode lets the model decide, which is a different workload. The lane's cluster template also exported UCX transport settings, which only a NIXL KV transfer would use (these workers have none), and the cluster's RDMA device names.\n", "json": "/MiniMaxAI/MiniMax-M3/presets/gb200_agentx.json" }, "gb200_agentx_tp8": { "label": "AgentX TP8", "hardware": "gb200", "strategy": "multi_node_tp", "variants": [ "nvfp4" ], "workload": "Agentic (InferenceX AgentX)", "description": "One TP8 worker across two GB200 trays, with the all-reduce over MNNVL and GPU-resident KV at concurrency 1 and 5; draft, kernels, graph mode and thinking as in AgentX TP4.", "source": { "label": "InferenceX #4281", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4281", "key": "minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp", "revision": "bbebca8b4be98a06401e943bad37874abbd65101" }, "image": { "vllm": "vllm/vllm-openai:nightly-5555a2d8f7116b2b4877f66516f4128176cf722a@sha256:8896fd7b29fc4ee7ce839f146cb4757984c476ec79d0234ffa8c995dee6e339a", "pip": [ "ai-dynamo-runtime==1.6.0.dev20261016", "ai-dynamo==1.6.0.dev20261016" ] }, "select": { "nodes": 2, "kv_offload": "off", "features": { "tool_calling": true, "reasoning": true, "spec_decoding": "eagle3_gqa", "text_only": true, "thinking_always_on": true, "encoder_parallel": false } }, "args": [ "--max-model-len", "1048576" ], "env": { "VLLM_ENGINE_READY_TIMEOUT_S": "7200", "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": "1800" }, "json_args": { "--compilation-config": { "cudagraph_mode": "FULL_AND_PIECEWISE" } }, "points": { "tp8_c1": { "label": "1", "concurrency": 1, "args": [ "--max-num-seqs", "2" ], "json_args": { "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 256, 512, 1024, 2048 ] } } }, "tp8_c5": { "label": "5", "concurrency": 5, "args": [ "--max-num-seqs", "8" ], "json_args": { "--compilation-config": { "cudagraph_capture_sizes": [ 1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 256, 512, 1024, 2048 ] } } } }, "runbook": "The TP8 points of the GB200 AgentX lane in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) (`minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp`, [SemiAnalysisAI/InferenceX#4281](https://github.com/SemiAnalysisAI/InferenceX/pull/4281)) run one aggregated vLLM worker across two 4-GPU trays with vLLM's multi-node tensor parallelism: the second tray runs the same command with its node rank and `--headless`. The KV cache stays in HBM. Both trays must be in the same NVL72 NVLink (IMEX) domain, because TP8's all-reduce runs over MNNVL.\n\nDraft, kernels, graph mode and engine environment match [AgentX TP4](#preset-gb200_agentx), whose runbook has the etcd, NATS, Dynamo worker and frontend steps; run the Dynamo worker on both trays. The InferenceX throughput points used synthetic rejection at the golden EAGLE3-GQA acceptance length (2.78); production serving keeps block verification, as in the command above.\n", "json": "/MiniMaxAI/MiniMax-M3/presets/gb200_agentx_tp8.json" }, "mi355x_agentx": { "label": "AgentX", "hardware": "mi355x", "strategy": "single_node_tp", "variants": [ "mxfp4" ], "workload": "Agentic (InferenceX AgentX)", "description": "MXFP4 with the GQA EAGLE3 draft (three tokens), thinking on and no offload; TP4 enables the pinned image's monokernel, while TP2 uses ordinary decode.", "source": { "label": "InferenceX #4418", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4418", "key": "minimaxm3-fp4-mi355x-vllm-agentic-mtp", "revision": "f10885621773d1f900b6b4d002ee37863e583af3" }, "image": "vllm/vllm-openai-rocm:nightly-df6603f0583375f244d1ac6d36265462b7490392@sha256:81899892d73777a4ae81e16f39e749169687f29249505eb4c56da83bd097d41c", "tp": 4, "select": { "kv_offload": "off", "features": { "tool_calling": true, "reasoning": true, "spec_decoding": "eagle3_gqa", "text_only": true, "thinking_always_on": true, "encoder_parallel": false } }, "remove_args": [ "--mm-encoder-tp-mode", "--mm-encoder-attn-backend" ], "args": [ "--served-model-name", "amd/MiniMax-M3-MXFP4", "--gpu-memory-utilization", "0.9", "--enable-chunked-prefill", "--max-num-batched-tokens", "32768", "--enable-prefix-caching", "--attention-backend", "ROCM_AITER_UNIFIED_ATTN", "--kv-cache-dtype", "fp8", "--stream-interval", "20", "--moe-backend", "aiter" ], "env": { "PYTHONNOUSERSITE": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "3600", "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS": "1800", "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16": "0", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB": "256" }, "json_args": { "--speculative-config": { "attention_backend": "ROCM_AITER_UNIFIED_ATTN" }, "--attention-config": { "indexer_kv_dtype": "fp8" }, "--compilation-config": { "cudagraph_mode": "FULL_DECODE_ONLY", "cudagraph_capture_sizes": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 48, 64, 72, 80, 88, 96, 104, 112, 120, 128 ] } }, "points": { "tp4_c1": { "label": "1 (TP4)", "concurrency": 1, "args": [ "--max-num-seqs", "2", "--monokernel" ] }, "tp4_c4": { "label": "4 (TP4)", "concurrency": 4, "args": [ "--max-num-seqs", "8", "--monokernel" ] }, "tp4_c5": { "label": "5 (TP4)", "concurrency": 5, "args": [ "--max-num-seqs", "10", "--monokernel" ] }, "tp4_c8": { "label": "8 (TP4)", "concurrency": 8, "args": [ "--max-num-seqs", "16", "--monokernel" ] }, "tp4_c10": { "label": "10 (TP4)", "concurrency": 10, "args": [ "--max-num-seqs", "20", "--monokernel" ] }, "tp4_c12": { "label": "12 (TP4)", "concurrency": 12, "args": [ "--max-num-seqs", "24", "--monokernel" ] }, "tp4_c15": { "label": "15 (TP4)", "concurrency": 15, "args": [ "--max-num-seqs", "30", "--monokernel" ] }, "tp4_c20": { "label": "20 (TP4)", "concurrency": 20, "args": [ "--max-num-seqs", "40", "--monokernel" ] }, "tp4_c24": { "label": "24 (TP4)", "concurrency": 24, "args": [ "--max-num-seqs", "48", "--monokernel" ] }, "tp4_c32": { "label": "32 (TP4)", "concurrency": 32, "args": [ "--max-num-seqs", "64", "--monokernel" ] }, "tp2_c1": { "label": "1 (TP2)", "concurrency": 1, "tp": 2, "args": [ "--max-num-seqs", "2" ] }, "tp2_c2": { "label": "2 (TP2)", "concurrency": 2, "tp": 2, "args": [ "--max-num-seqs", "4" ] } }, "runbook": "This opt-in MXFP4 profile follows [InferenceX #4418](https://github.com/SemiAnalysisAI/InferenceX/pull/4418)\nat `f10885621773d1f900b6b4d002ee37863e583af3`: the\n[`agg-variants.yaml` SRT bundle](https://github.com/SemiAnalysisAI/InferenceX/blob/f10885621773d1f900b6b4d002ee37863e583af3/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/vllm/mi355x-fp4-mtp/agg-variants.yaml)\nfor `minimaxm3-fp4-mi355x-vllm-agentic-mtp`. It serves\n[`amd/MiniMax-M3-MXFP4`](https://huggingface.co/amd/MiniMax-M3-MXFP4) on one\nMI355X node, text-only, with no expert parallelism, model offload or KV offload.\nBoth TP4 and TP2 use Model Runner V2; every point sets `max-num-seqs` to twice\nits client concurrency. INT4 QuickReduce, FP8 KV/indexer caches, the 0.9 memory\nfraction and the 128-token graph limit are specific to this workload.\n\nTP4 enables the [pinned `df6603f` monokernel](https://github.com/vllm-project/vllm/blob/df6603f0583375f244d1ac6d36265462b7490392/vllm/models/minimax_m3/amd/specialized_decode/runtime/spec.py).\nIt handles supported physical decode widths 1–16 with EAGLE3 K3; larger or\nmixed steps fall back to ordinary decode. TP2 omits `--monokernel` because\nthis implementation requires TP4. The profile reproduces that image, not\nlater monokernel implementations or general MiniMax-M3 defaults.\n\n[Source run 39890471932](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/39890471932)\ncontains the twelve throughput points and their real-verification evals.\nThe audited head's bound serving configurations match that run's source.\n[Reuse run 40416678647](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40416678647)\nskipped the GPU matrices; it is not a fresh qualification. The earlier\n[#2959 baseline](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/34609443618/attempts/2)\ndoes not qualify this image or monokernel. Performance is reported by\nInferenceX; local recipe validation does not reproduce GPU performance.\n\nOrdinary serving and accuracy evaluation retain real target verification.\nThroughput reproduction alone adds the following keys to `--speculative-config`:\n\n```json\n{\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":2.78}\n```\n\nThe [golden 2.78 acceptance length](https://github.com/SemiAnalysisAI/InferenceX/blob/f10885621773d1f900b6b4d002ee37863e583af3/inferencex-e2e/infx/golden_al_distribution/minimaxm3_eagle3_gqa.yaml)\nis for the three-token GQA draft with thinking enabled. Synthetic rejection\nbypasses real verification; never use it for users or accuracy evals.\nAgentX's `AIPERF_APPLY_CHAT_TEMPLATE=true` and required `vllm:` metrics prefix\nare benchmark-client settings, not server flags. The source's\n`dram-utilization: 0.20` is orchestration metadata, not KV offload.\n", "json": "/MiniMaxAI/MiniMax-M3/presets/mi355x_agentx.json" } }, "notes": [ { "title": "Host KV offload on GB200", "when": { "hardware": [ "gb200" ] }, "body": "By default, **KV Offload → Simple** reserves 220 GiB of pinned host\nmemory per rank (880 GiB per 4-GPU tray) with `lazy_offload: false`,\nmore than a GB200 tray can give KV blocks. Lower\n`cpu_bytes_to_use_per_rank` in its `--kv-transfer-config`: the\n[AgentX TP4](#preset-gb200_agentx) KV offload points use 194347270144\n(181 GiB) per rank with `lazy_offload: true`.\n" } ], "guide": "## Overview\n\n[MiniMax-M3](https://huggingface.co/MiniMaxAI/MiniMax-M3) is a frontier\nvision-language MoE model from [MiniMax](https://www.minimax.io/).\n\n- **MSA (MiniMax Sparse Attention)** — scalable sparse-attention architecture\n that lifts the context window to **1M tokens**. MiniMax reports per-token\n compute at 1M context reduced to ~1/20 of the previous generation, with\n >9× prefill and >15× decode speedup vs dense baselines.\n- **Frontier coding and agent capabilities** — SWE-Bench Pro 59.0%,\n Terminal-Bench 2.1 66.0%, SWE-fficiency 34.8%, KernelBench Hard 28.8%,\n MCP Atlas 74.2%.\n- **Native multimodal** — image + video inputs, plus computer-use; trained\n multimodally from step 0.\n- **Two reasoning modes** — `thinking` (complex reasoning / agents) and\n `non-thinking` (latency-sensitive), switchable per request.\n\n## Prerequisites\n\n- **OS:** Linux\n- **Python:** 3.10 - 3.13\n- **NVIDIA:** compute capability >= 9.0 (Hopper) recommended; 8x H200 / H20\n for a tight single-node BF16 fit, or multi-node TP for long-context headroom\n- **AMD:** MI350X/MI355X (gfx950), MI300X/MI325X (gfx942), ROCm 7.2+. BF16\n needs TP=8; the MXFP8 variant runs from TP=4.\n- **`--block-size 128` is mandatory on every platform** (MSA sparse/index cache).\n\n### Docker (NVIDIA)\n\nMiniMax-M3 support has not yet shipped in a stable vLLM release — use the\ndedicated Docker image:\n\n```bash\ndocker pull vllm/vllm-openai:minimax-m3\n```\n\n### Docker (AMD ROCm)\n\nUse the nightly image, which is the image the AMD variants of this recipe pin:\n\n```bash\ndocker pull vllm/vllm-openai-rocm:nightly\n```\n\n```bash\ndocker run --rm -it --device /dev/kfd --device /dev/dri --group-add video \\\n --cap-add SYS_PTRACE --security-opt seccomp=unconfined --ipc=host \\\n --shm-size=16g -p 8000:8000 \\\n --entrypoint /bin/bash \\\n vllm/vllm-openai-rocm:nightly\n```\n\n## Launching the Server\n\n### NVIDIA — TP8 (8x H200 / H20)\n\n```bash\nvllm serve MiniMaxAI/MiniMax-M3 \\\n --tensor-parallel-size 8 \\\n --block-size 128 \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\n#### TP8 + Expert Parallel\n\n```bash\nvllm serve MiniMaxAI/MiniMax-M3 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --block-size 128 \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\n#### DP8 + Expert Parallel\n\n```bash\nvllm serve MiniMaxAI/MiniMax-M3 \\\n --data-parallel-size 8 \\\n --enable-expert-parallel \\\n --block-size 128 \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\n### AMD ROCm (MI350X/MI355X (gfx950), MI300X/MI325X (gfx942))\n\nOn AMD MI300X / MI325X / MI355X, run with CUDA graphs and set the following\nbefore any of the serve commands below. It avoids the MiniMax-M3 decode breakable-cudagraph path that would\notherwise force eager execution (per @hongxiayang):\n\n```bash\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=0\n```\n\nFor gfx950: Prefer using the MXFP8 variant `MiniMaxAI/MiniMax-M3-MXFP8` for TP=4 and a smaller\nfootprint. Use TP=8 for lower latency or long context length, or the default bf16 model.\n\n#### TP8 (Text or Vision)\n\n```bash\nvllm serve MiniMaxAI/MiniMax-M3 \\\n --tensor-parallel-size 8 \\\n --block-size 128 \\\n --attention-backend TRITON_ATTN \\\n --mm-encoder-tp-mode data \\\n --mm-encoder-attn-backend ROCM_AITER_FA \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\nAdd the vision-encoder flags (`--mm-encoder-tp-mode data\n--mm-encoder-attn-backend ROCM_AITER_FA`) for multimodal serving.\n\n#### TP8 + `--quantization fp8_per_channel`\n\nOnline **per-channel FP8 (PTPC)** quantization of the BF16 checkpoint halves the\nweight footprint (≈100 → **≈50 GiB/GPU**, ~1.75× more KV cache) and lifts\nbatched throughput with **gsm8k unchanged from BF16** (lossless); \nThese env vars and the flag are for the **BF16 checkpoint only**.\n\n```bash\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=0\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_USE_AITER_MOE=0\n\nvllm serve MiniMaxAI/MiniMax-M3 \\\n --tensor-parallel-size 8 \\\n --block-size 128 \\\n --attention-backend TRITON_ATTN \\\n --quantization fp8_per_channel \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\n#### TP8 + Expert Parallel\n\n```bash\nvllm serve MiniMaxAI/MiniMax-M3 \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel \\\n --block-size 128 \\\n --attention-backend TRITON_ATTN \\\n --mm-encoder-tp-mode data \\\n --mm-encoder-attn-backend ROCM_AITER_FA \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\n#### DP8 + Expert Parallel\n\n```bash\nvllm serve MiniMaxAI/MiniMax-M3 \\\n --data-parallel-size 8 \\\n --enable-expert-parallel \\\n --block-size 128 \\\n --attention-backend TRITON_ATTN \\\n --mm-encoder-tp-mode data \\\n --mm-encoder-attn-backend ROCM_AITER_FA \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\n## FP8 KV Cache\n\nAdd `--kv-cache-dtype fp8` to any command for ~1.5× the KV pool — lossless\nin our testing across the full native context. Especially worth it for high\nconcurrency or long context, where KV is the binding constraint.\n\n## Context Length & GPU Memory\n\nThe full **1M-token** window (`context_length: 1048576`) needs a large KV\ncache. To save GPU memory, you can optionally cap the context with\n`--max-model-len`:\n\n```bash\nvllm serve MiniMaxAI/MiniMax-M3 \\\n --tensor-parallel-size 8 \\\n --block-size 128 \\\n --max-model-len 131072 # 128K instead of the full 1M\n```\n\nAMD ROCm notes: Native context is 512K. To go past it, supply a YaRN rope_scaling on the text config\n(a top-level override silently misses the decoder's config) and allow the long max length.\nTP=8 + fp8 KV is the practical combo at 1M:\n\n```bash\nVLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \\\n vllm serve MiniMaxAI/MiniMax-M3 \\\n --block-size 128 \\\n --kv-cache-dtype fp8 \\\n --tensor-parallel-size 8 \\\n --max-model-len 1048576 \\\n --attention-backend TRITON_ATTN \\\n --mm-encoder-tp-mode data \\\n --mm-encoder-attn-backend ROCM_AITER_FA \\\n --tool-call-parser minimax_m3 \\\n --enable-auto-tool-choice \\\n --reasoning-parser minimax_m3 \\\n --hf-overrides '{\"text_config\":{\"rope_scaling\":{\"rope_type\":\"yarn\",\"factor\":2.0,\"original_max_position_embeddings\":524288}}}'\n```\n\n- Set `--max-model-len` to the longest prompt + output you actually serve\n (e.g. `32768`, `131072`, `262144`). A smaller window frees KV-pool headroom\n for higher concurrency and lets the model fit on fewer GPUs; if you need the\n full 1M window, consider scaling out with multi-node TP instead.\n\n## Client Usage\n\nRecommended sampling parameters (from the model card):\n\n- `temperature = 1.0`\n- `top_p = 0.95`\n- `top_k = 40`\n\nDefault system prompt:\n\n```\nYou are a helpful assistant. Your name is MiniMax-M3 and is built by MiniMax.\n```\n\nExample chat request:\n\n```bash\ncurl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"MiniMaxAI/MiniMax-M3\",\n \"temperature\": 1.0,\n \"top_p\": 0.95,\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant. Your name is MiniMax-M3 and is built by MiniMax.\"},\n {\"role\": \"user\", \"content\": \"Explain MSA sparse attention in 3 bullets.\"}\n ]\n }'\n```\n\n## Thinking Modes\n\nM3 reasoning is controlled by the `thinking_mode`, there are three values:\n\n- **`enabled`** — the model thinks before every response, including after tool results. \n Use for complex reasoning and agents.\n- **`disabled`** — no thinking; the model answers directly. Use for\n latency-sensitive turns.\n- **`adaptive`** *(default when unset)* — the model decides whether to think\n based on the task.\n\nPass it per request through `chat_template_kwargs`. The same value also tunes\nthe `minimax_m3` reasoning parser, so `reasoning` and `content` are\nsplit correctly in every mode.\n\n```python\n\n# Start the MiniMax-M3 model by referring to the command above first.\n\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\n\nresponse = client.chat.completions.create(\n model=\"MiniMaxAI/MiniMax-M3\",\n messages=[{\"role\": \"user\", \"content\": \"Prove there are infinitely many primes.\"}],\n extra_body={\"chat_template_kwargs\": {\"thinking_mode\": \"enabled\"}},\n)\nmsg = response.choices[0].message\nprint(msg.reasoning) # the block\nprint(msg.content) # the final answer\n```\n\n## Benchmarking\n\n```bash\nvllm bench serve \\\n --backend vllm \\\n --model MiniMaxAI/MiniMax-M3 \\\n --endpoint /v1/completions \\\n --dataset-name random \\\n --random-input 2048 \\\n --random-output 1024 \\\n --max-concurrency 10 \\\n --num-prompt 100\n```\n\n## Quantized Variant (MXFP8)\n\n[`MiniMaxAI/MiniMax-M3-MXFP8`](https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8)\nis an MXFP8 checkpoint quantized by NVIDIA from the original FP16 weights —\nroughly half the VRAM of the BF16 release. Select the **mxfp8** variant above,\nor pass the repo id directly to `vllm serve`:\n\n```bash\nvllm serve MiniMaxAI/MiniMax-M3-MXFP8 \\\n --tensor-parallel-size 8 \\\n --block-size 128 \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\nFor best MXFP8 throughput, prefer Blackwell (B200/B300) for native MX tensor cores,\nor AMD CDNA4 (MI350X/MI355X, gfx950) for native MXFP8 matrix cores.\n\nOn AMD MI300X, the **mxfp8** variant stacks two accuracy-safe scheduling\nlevers from the InferenceX MI300X tuning recipe and avoids the high-concurrency\nexpert-parallel regression by using plain TP8:\n\n```bash\nexport VLLM_ENGINE_READY_TIMEOUT_S=3600\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=0\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_USE_AITER_MHA=0\nexport TORCH_BLAS_PREFER_HIPBLASLT=1\nexport NCCL_MIN_NCHANNELS=\"${NCCL_MIN_NCHANNELS:-112}\"\nexport GPU_MAX_HW_QUEUES=\"${GPU_MAX_HW_QUEUES:-2}\"\n\nvllm serve MiniMaxAI/MiniMax-M3-MXFP8 \\\n --tensor-parallel-size 8 \\\n --block-size 128 \\\n --language-model-only \\\n --max-model-len \"$MAX_MODEL_LEN\" \\\n --attention-backend TRITON_ATTN \\\n --async-scheduling \\\n --max-num-batched-tokens 16384 \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\nOn AMD MI355X the **mxfp8** variant additionally enables the AITER kernels,\nthe fused shared-experts MoE, and INT4 QuickReduce, and pulls the nightly ROCm\nimage (`vllm/vllm-openai-rocm:nightly`) — selected automatically in the command\nbuilder. If you launch by hand:\n\n```bash\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=0\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\n\nvllm serve MiniMaxAI/MiniMax-M3-MXFP8 \\\n --tensor-parallel-size 8 \\\n --block-size 128 \\\n --attention-backend TRITON_ATTN \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\n### MXFP8 high-concurrency options on MI355X\n\nThe command above is the default serving configuration and is tuned for the\ngeneral case. **AITER page-16 sparse paged attention** helps specifically at **long context and\nhigh concurrency**. It maps MiniMax-M3's top-k 128-token MSA blocks onto AITER page-16 block tables\nand runs AITER Gluon paged attention over only the selected KV pages.\n\nThe high-concurrency configuration is:\n\n```bash\n# Long context + high concurrency only (isl >= 8k, conc >= 64). Regresses\n# short-context / low-concurrency serving.\nexport VLLM_USE_BREAKABLE_CUDAGRAPH=0\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1\nexport VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\nexport VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256\n\nvllm serve MiniMaxAI/MiniMax-M3-MXFP8 \\\n --tensor-parallel-size 4 \\\n --block-size 128 \\\n --language-model-only \\\n --moe-backend aiter \\\n --kv-cache-dtype fp8 \\\n --max-num-batched-tokens 32768 \\\n --attention-backend TRITON_ATTN \\\n --linear-backend emulation \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\nNote: Outside the long-context/high-batch\nregime the shuffled layout costs more than it saves; leave it at `0`:\n\n```bash\nexport VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0\n```\n\n## Quantized Variant (NVFP4 on Blackwell)\n\n[`nvidia/MiniMax-M3-NVFP4`](https://huggingface.co/nvidia/MiniMax-M3-NVFP4)\nis an NVFP4 checkpoint quantized by NVIDIA with ModelOpt. It requires a vLLM\nbuild with MiniMax-M3 NVFP4 support (vllm-project/vllm PR #46380).\n\nSelect the **nvfp4** variant and **B200** hardware in the command builder, or\nlaunch the tested B200 baseline directly:\n\n```bash\nVLLM_FLOAT32_MATMUL_PRECISION=high \\\n vllm serve nvidia/MiniMax-M3-NVFP4 \\\n --tensor-parallel-size 8 \\\n --trust-remote-code \\\n --block-size 128 \\\n --tool-call-parser minimax_m3 \\\n --reasoning-parser minimax_m3 \\\n --enable-auto-tool-choice\n```\n\nPair it with EAGLE3 speculative decoding (the *Spec decoding* feature) for\nfaster decode — the validated B200 benchmark runs the NVFP4 target against the\n`Inferact/MiniMax-M3-EAGLE3` draft head (3 speculative tokens):\n\n```bash\nVLLM_FLOAT32_MATMUL_PRECISION=high \\\n vllm serve nvidia/MiniMax-M3-NVFP4 \\\n --tensor-parallel-size 8 \\\n --trust-remote-code \\\n --block-size 128 \\\n --language-model-only \\\n --speculative-config '{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}'\n```\n\nFor long-context or high-batch workloads where draft KV memory is the binding\nconstraint, swap in the **GQA** variant:\n\n```bash\nVLLM_FLOAT32_MATMUL_PRECISION=high \\\n vllm serve nvidia/MiniMax-M3-NVFP4 \\\n --tensor-parallel-size 8 \\\n --trust-remote-code \\\n --block-size 128 \\\n --language-model-only \\\n --speculative-config '{\"method\": \"eagle3\", \"model\": \"Inferact/MiniMax-M3-EAGLE3-GQA\", \"num_speculative_tokens\": 3, \"attention_backend\": \"FLASH_ATTN\"}'\n```\n\nYou may also consider FlashInfer for draft attention if it performs better for\nyour workload, and local argmax reduction to reduce communication traffic:\n\n```bash\n--speculative-config '{\"method\":\"eagle3\",\"model\":\"Inferact/MiniMax-M3-EAGLE3-GQA\",\"num_speculative_tokens\":3,\"attention_backend\":\"FLASHINFER\",\"use_local_argmax_reduction\":true}'\n```\n\n## GB200: InferenceX AgentX aggregated (Dynamo + vLLM)\n\nThe GB200 AgentX lane in [SemiAnalysis InferenceX](https://inferencex.semianalysis.com/)\n(`minimaxm3-fp4-gb200-dynamo-vllm-agentic-agg-mtp`,\n[SemiAnalysisAI/InferenceX#4281](https://github.com/SemiAnalysisAI/InferenceX/pull/4281))\nserves the NVFP4 checkpoint with the GQA EAGLE3 draft as aggregated vLLM\nworkers behind the Dynamo frontend's KV-aware router. Its measured points are\ntwo opt-in presets: [AgentX TP4](#preset-gb200_agentx), one worker per 4-GPU\ntray, whose larger points add host KV offload; and\n[AgentX TP8](#preset-gb200_agentx_tp8), one worker across two trays. Each\npoint renders the worker's engine flags, environment and pinned image. The\nTP4 preset's runbook has the etcd, NATS, Dynamo worker and frontend steps\nthat the command builder doesn't render.\n\n## Quantized Variant (MXFP4 on MI355X)\n\n[`amd/MiniMax-M3-MXFP4`](https://huggingface.co/amd/MiniMax-M3-MXFP4)\nuses static OCP MXFP4 weights and dynamic OCP MXFP4 activations produced with\nAMD Quark. It targets AMD Instinct MI355X (gfx950) and requires a ROCm\nnightly containing the Quark MXFP4 fix from vLLM PR #45794.\n\nSelect the **mxfp4** variant and **MI355X** hardware in the command builder,\nor launch the tested TP8 baseline directly:\n\n```bash\nvllm serve amd/MiniMax-M3-MXFP4 \\\n --tensor-parallel-size 8 \\\n --trust-remote-code \\\n --block-size 128 \\\n --attention-backend TRITON_ATTN \\\n --mm-encoder-tp-mode data \\\n --mm-encoder-attn-backend ROCM_AITER_FA \\\n --tool-call-parser minimax_m3 \\\n --enable-auto-tool-choice \\\n --reasoning-parser minimax_m3\n```\n\n### MXFP4 AgentX on MI355X\n\nThe opt-in [AgentX preset](#preset-mi355x_agentx) follows\n[InferenceX #4418](https://github.com/SemiAnalysisAI/InferenceX/pull/4418) on its\npinned ROCm image: TP4 at concurrency 1, 4, 5, 8, 10, 12, 15, 20, 24 and 32\nenables monokernel; TP2 at 1 and 2 uses ordinary decode. Both use Model Runner V2,\nthe three-token GQA EAGLE3 draft and real target verification, with no offload.\nThese text-only MXFP4 settings are workload-specific; the general serving\nconfiguration remains unchanged. Synthetic acceptance is for throughput\nreproduction only, as described in the preset's runbook.\n\n## Troubleshooting\n\n- **`--block-size` mismatch.** MSA's sparse block size is 128; the vLLM KV\n cache block size must match. Using the default (16) misaligns the sparse\n attention indexing (on AMD it crashes with `No common block size for 16`).\n- **Parsers.** `--tool-call-parser` and `--reasoning-parser` both use\n `minimax_m3` — distinct from `minimax_m2` used by earlier releases.\n- **Long context KV cache.** See *Context Length & GPU Memory* above — cap\n `--max-model-len` or scale to multi-node TP if you OOM.\n- **Vision encoder.** The encoder is small, so at high TP the *Encoder\n Parallel* option runs it data-parallel (`--mm-encoder-tp-mode data`) to\n avoid TP comm overhead; it also turns on the vision-encoder attention\n backend (FlashInfer on NVIDIA, `--mm-encoder-attn-backend FLASHINFER`;\n AITER FlashAttention on AMD, `ROCM_AITER_FA`) and the host-shared-memory\n multimodal processor cache (`--mm-processor-cache-type shm`). For **text-only** workloads enable *Text only*\n (`--language-model-only`) to skip loading the encoder and free VRAM — it is\n mutually exclusive with Encoder Parallel.\n\n\n## EAGLE3 Speculative Decoding\n\nTwo Inferact draft heads are available as selectable modes in the *Spec decoding*\nfeature — both use `method: eagle3` with 3 speculative tokens and share\nMiniMax-M3's embedding table and LM head:\n\n| Mode | Repo | KV heads | Draft KV cache |\n|------|------|----------|----------------|\n| **MHA** (default) | `Inferact/MiniMax-M3-EAGLE3` | 64 | baseline |\n| **GQA** | `Inferact/MiniMax-M3-EAGLE3-GQA` | 4 | **16× smaller** |\n\nThe GQA head uses Grouped Query Attention (4 KV heads grouped from 64 query\nheads) — the only architectural difference from the MHA head. The 16× reduction\nin draft KV state makes it preferable when draft memory is the binding constraint\n(long-context serving or large batch sizes).\n\n## References\n\n- [Model card](https://huggingface.co/MiniMaxAI/MiniMax-M3)\n- [MXFP8 variant](https://huggingface.co/MiniMaxAI/MiniMax-M3-MXFP8)\n- [NVFP4 variant](https://huggingface.co/nvidia/MiniMax-M3-NVFP4)\n- [AMD MXFP4 variant](https://huggingface.co/amd/MiniMax-M3-MXFP4)\n- [EAGLE3 draft head (MHA)](https://huggingface.co/Inferact/MiniMax-M3-EAGLE3)\n- [EAGLE3 draft head (GQA)](https://huggingface.co/Inferact/MiniMax-M3-EAGLE3-GQA)\n- [MiniMax](https://www.minimax.io/)\n- [MiniMax Agent](https://agent.minimax.io/)\n- [MiniMax Platform](https://platform.minimax.io/)\n- [Previous release: MiniMax-M2.7](https://huggingface.co/MiniMaxAI/MiniMax-M2.7)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/MiniMaxAI/MiniMax-M3.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/MiniMaxAI/MiniMax-M3.yaml", "overlays": [] } }