{ "hf_id": "deepseek-ai/DeepSeek-V4.1-Flash", "meta": { "title": "DeepSeek-V4.1-Flash", "slug": "deepseek-v4.1-flash", "provider": "DeepSeek", "description": "DeepSeek V4.1 Flash vision-language MoE (552B backbone; 8B active per prompt token, 16B per output token) combining sliding-window plus compressed sparse attention with a two-level indexer, engram n-gram memory, hyper-connections, and a DSpark multi-token draft head.", "date_added": "2026-09-09", "date_updated": "2026-10-25", "tasks": [ "text", "multimodal" ], "performance_headline": "1M context at 8-16B active parameters, with engram n-gram memory", "related_recipes": [ "deepseek-ai/DeepSeek-V4-Flash", "deepseek-ai/DeepSeek-V4-Pro" ], "default_hardware": "gb200", "hardware": { "h100": "verified", "h200": "verified", "b200": "verified", "gb200": "verified", "gb300": "verified", "b300": "verified", "mi300x": "verified", "mi325x": "verified", "mi350x": "verified", "mi355x": "verified", "dgx_station_gb300": "verified" } }, "recommended_command": { "hardware": "gb200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_ENGINE_READY_TIMEOUT_S": "3600", "VLLM_USE_RUST_FRONTEND": "1" }, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve deepseek-ai/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-mode deepseek_v41 \\\n --tensor-parallel-size 2 \\\n --attention-config '{\"backend\":\"FLASHINFER_MLA_SPARSE_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}' \\\n --kv-cache-dtype fp8 \\\n --max-num-seqs 128 \\\n --engram-config '{\"cpu_offload\":true}' \\\n --tool-call-parser deepseek_v41 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v41 \\\n --mm-encoder-tp-mode data", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4.1-Flash-1017", "--tokenizer-mode", "deepseek_v41", "--tensor-parallel-size", "2", "--attention-config", "{\"backend\":\"FLASHINFER_MLA_SPARSE_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}", "--kv-cache-dtype", "fp8", "--max-num-seqs", "128", "--engram-config", "{\"cpu_offload\":true}", "--tool-call-parser", "deepseek_v41", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v41", "--mm-encoder-tp-mode", "data" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n vllm/vllm-openai:latest deepseek-ai/DeepSeek-V4.1-Flash-1017 \\\n --tokenizer-mode deepseek_v41 \\\n --tensor-parallel-size 2 \\\n --attention-config '{\"backend\":\"FLASHINFER_MLA_SPARSE_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}' \\\n --kv-cache-dtype fp8 \\\n --max-num-seqs 128 \\\n --engram-config '{\"cpu_offload\":true}' \\\n --tool-call-parser deepseek_v41 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v41 \\\n --mm-encoder-tp-mode data", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=3600", "-e", "VLLM_USE_RUST_FRONTEND=1", "vllm/vllm-openai:latest", "deepseek-ai/DeepSeek-V4.1-Flash-1017", "--tokenizer-mode", "deepseek_v41", "--tensor-parallel-size", "2", "--attention-config", "{\"backend\":\"FLASHINFER_MLA_SPARSE_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}", "--kv-cache-dtype", "fp8", "--max-num-seqs", "128", "--engram-config", "{\"cpu_offload\":true}", "--tool-call-parser", "deepseek_v41", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v41", "--mm-encoder-tp-mode", "data" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "blackwell", "compute_arch": "sm100", "display_name": "GB200 NVL4", "description": "NVIDIA GB200 Grace-Blackwell compute tray · 192 GB HBM3e/GPU · 4 Blackwell GPUs (NVL72 tray unit)", "gpu_count": 4, "vram_gb": 768, "multi_node": false }, "alternatives": { "single_node_tep": "/deepseek-ai/DeepSeek-V4.1-Flash/strategies/single_node_tep.json", "single_node_dep": "/deepseek-ai/DeepSeek-V4.1-Flash/strategies/single_node_dep.json", "multi_node_tp": "/deepseek-ai/DeepSeek-V4.1-Flash/strategies/multi_node_tp.json", "multi_node_tep": "/deepseek-ai/DeepSeek-V4.1-Flash/strategies/multi_node_tep.json", "multi_node_dep": "/deepseek-ai/DeepSeek-V4.1-Flash/strategies/multi_node_dep.json", "pd_cluster": "/deepseek-ai/DeepSeek-V4.1-Flash/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/deepseek-ai/DeepSeek-V4.1-Flash/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/deepseek-ai/DeepSeek-V4.1-Flash/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "gb200": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/gb200.json", "h100": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/h100.json", "h200": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/h200.json", "b200": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/b200.json", "b300": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/b300.json", "gb300": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/gb300.json", "dgx_station_gb300": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/dgx_station_gb300.json", "mi300x": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/mi300x.json", "mi325x": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/mi325x.json", "mi350x": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/mi350x.json", "mi355x": "/deepseek-ai/DeepSeek-V4.1-Flash/hw/mi355x.json" } }, "model": { "model_id": "deepseek-ai/DeepSeek-V4.1-Flash", "smg_grpc": { "reasoning_parser": "deepseek_v41", "tool_call_parser": "deepseek_v41" }, "min_vllm_version": "0.31.0", "default_frontend": "rust", "docker_image": { "amd": "vllm/vllm-openai-rocm:nightly-rocm100" }, "architecture": "moe", "parameter_count": "552B", "active_parameters": "8-16B", "context_length": 1048576, "base_args": [ "--tokenizer-mode", "deepseek_v41" ], "base_env": { "VLLM_ENGINE_READY_TIMEOUT_S": "3600" }, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable tool calling. V4.1 emits DSML tag blocks rather than JSON fences.", "args": [ "--tool-call-parser", "deepseek_v41", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "Enable thinking mode with the DeepSeek V4.1 reasoning parser.", "args": [ "--reasoning-parser", "deepseek_v41" ] }, "spec_decoding": { "description": "DSpark multi-token drafting, 5 tokens per round (the trained block size). On NVIDIA, adaptive verification scores each (request, position) slot and only high-scoring slots are verified, up to a startup-profiled compute budget. On AMD, vLLM currently refuses that flag, so the generated command sets enable_adaptive_verification:false and verifies the full 5-token block. Remove the AMD override only after a serve with the true flag actually starts on ROCm.", "args": [ "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"block\",\"enable_adaptive_verification\":true}" ], "hardware_overrides": { "amd": { "args": [ "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"rejection_sample_method\":\"block\",\"enable_adaptive_verification\":false}" ] }, "dgx_station_gb300": { "args": [ "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"num_speculative_tokens_per_batch_size\":[[1,4,5],[5,24,1]]}" ] } } }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "encoder_parallel": { "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.", "args": [ "--mm-encoder-tp-mode", "data" ] } }, "opt_in_features": [ "spec_decoding", "text_only" ], "variants": { "default": { "model_id": "deepseek-ai/DeepSeek-V4.1-Flash-1017", "label": "FP8 (1017)", "precision": "fp8", "vram_minimum_gb": 614, "description": "DeepSeek-V4.1-Flash-1017 (2026-10-17): native mixed checkpoint (MXFP4 MoE experts, MXFP8 block-quantized dense weights, UE8M0 scales) with updated weights — stronger reasoning, coding, agentic and visual results, and shorter outputs.", "hardware_overrides": { "h100": { "extra_args": [ "--engram-config", "{\"cpu_offload\":true}", "--max-num-batched-tokens", "4096", "--gpu-memory-utilization", "0.92" ], "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1", "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True" } }, "mi325x": { "install": { "pip": false } }, "mi355x": { "install": { "pip": false } }, "dgx_station_gb300": { "vram_minimum_gb": 244, "extra_args": [ "--offload-backend", "uva", "--cpu-offload-gb", "60", "--cpu-offload-params", "routed_experts.w13_weight", "routed_experts.w2_weight", "--engram-config", "{\"cpu_offload\":true}", "--gpu-memory-utilization", "0.97", "--max-model-len", "1048576", "--max-num-batched-tokens", "8192", "--long-prefill-token-threshold", "6144", "--compilation-config", "{\"cudagraph_capture_sizes\":[1,2,4,6,8,12,16,18,24,32,40,48,64,96,128]}" ] } }, "json": "/deepseek-ai/DeepSeek-V4.1-Flash-1017.json" }, "fp8": { "label": "FP8 (0909)", "precision": "fp8", "vram_minimum_gb": 614, "description": "Launch checkpoint (2026-09-09), superseded by 1017 — kept for reproducing earlier results. Same mixed MXFP4/MXFP8 format.", "hardware_overrides": { "h100": { "extra_args": [ "--engram-config", "{\"cpu_offload\":true}", "--max-num-batched-tokens", "4096", "--gpu-memory-utilization", "0.92" ], "extra_env": { "VLLM_USE_V2_MODEL_RUNNER": "1", "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True" } }, "mi325x": { "install": { "pip": false } }, "mi355x": { "install": { "pip": false } }, "dgx_station_gb300": { "vram_minimum_gb": 244, "extra_args": [ "--offload-backend", "uva", "--cpu-offload-gb", "60", "--cpu-offload-params", "routed_experts.w13_weight", "routed_experts.w2_weight", "--engram-config", "{\"cpu_offload\":true}", "--gpu-memory-utilization", "0.97", "--max-model-len", "1048576", "--max-num-batched-tokens", "8192", "--long-prefill-token-threshold", "6144", "--compilation-config", "{\"cudagraph_capture_sizes\":[1,2,4,6,8,12,16,18,24,32,40,48,64,96,128]}" ] } } } }, "kv_offload_support": { "offloading_cpu": "verified", "offloading_fs": "verified" }, "strategy_hardware": { "single_node_tep": { "dgx_station_gb300": "unsupported" }, "single_node_dep": { "dgx_station_gb300": "unsupported" } }, "hardware_overrides": { "amd": { "extra_args": [ "--gpu-memory-utilization", "0.9", "--moe-backend", "aiter" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "AITER_TRITON_LOG_LEVEL": "ERROR", "VLLM_USE_BREAKABLE_CUDAGRAPH": "1" } } }, "presets": { "b200_agentx_tp": { "label": "AgentX TP", "hardware": "b200", "strategy": "single_node_tp", "variants": [ "fp8", "default" ], "workload": "Agentic (InferenceX AgentX)", "default": true, "description": "TP4 for concurrency 1 to 128 and TP2 at concurrency 1, with THP-backed Engram tables, FlashInfer autotuning, 0.97 GPU memory and graph sizes in DSpark verification multiples.", "source": { "label": "InferenceX #3837", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/3837", "key": "dsv41flash-fp4-b200-vllm-agentic-dspark", "revision": "f817514" }, "image": "vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7", "select": { "frontend": "rust", "features": { "spec_decoding": true, "text_only": true, "encoder_parallel": false } }, "args": [ "--kernel-config", "{\"enable_flashinfer_autotune\":true}", "--gpu-memory-utilization", "0.97", "--engram-config", "{\"cpu_offload\":true,\"use_thp\":true}" ], "env": { "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "7200" }, "json_args": { "--speculative-config": { "enable_adaptive_verification": false } }, "points": { "tp2_c1": { "label": "1 (TP2)", "concurrency": 1, "default": true, "tp": 2, "args": [ "--max-num-seqs", "8", "--max-num-batched-tokens", "4096", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092]}", "--max-cudagraph-capture-size", "4092" ] }, "tp4_c1": { "label": "1 (TP4)", "concurrency": 1, "tp": 4, "args": [ "--max-num-seqs", "8", "--max-num-batched-tokens", "4096", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092]}", "--max-cudagraph-capture-size", "4092" ] }, "tp4_c4": { "label": "4 (TP4)", "concurrency": 4, "tp": 4, "args": [ "--max-num-seqs", "8", "--max-num-batched-tokens", "4096", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092]}", "--max-cudagraph-capture-size", "4092" ] }, "tp4_c8": { "label": "8 (TP4)", "concurrency": 8, "tp": 4, "args": [ "--max-num-seqs", "16", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}", "--max-cudagraph-capture-size", "8190" ] }, "tp4_c16": { "label": "16 (TP4)", "concurrency": 16, "tp": 4, "args": [ "--max-num-seqs", "32", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048,2304,2560,2816,3072,3328,3584,3840,4096]}", "--max-cudagraph-capture-size", "8190" ] }, "tp4_c32": { "label": "32 (TP4)", "concurrency": 32, "tp": 4, "args": [ "--max-num-seqs", "128", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048,2304,2560,2816,3072,3328,3584,3840,4096]}", "--max-cudagraph-capture-size", "8190" ] }, "tp4_c128": { "label": "128 (TP4)", "concurrency": 128, "tp": 4, "args": [ "--max-num-seqs", "256", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048,2304,2560,2816,3072,3328,3584,3840,4096]}", "--max-cudagraph-capture-size", "8190" ] } }, "json": "/deepseek-ai/DeepSeek-V4.1-Flash/presets/b200_agentx_tp.json" }, "b200_agentx_dep": { "label": "AgentX DEP", "hardware": "b200", "strategy": "single_node_dep", "variants": [ "fp8", "default" ], "workload": "Agentic (InferenceX AgentX)", "default": true, "description": "DEP2 for concurrency 8 to 32, DEP2 with a host-DRAM KV tier at 64 and 128, and DEP4 at 96 and 128: FlashInfer sparse MLA, MegaMoE and Engram tables in host memory, behind a consistent-hash vLLM Router.", "source": { "label": "InferenceX #3837", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/3837", "key": "dsv41flash-fp4-b200-vllm-agentic-dspark", "revision": "f817514" }, "image": "vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7", "select": { "frontend": "rust", "features": { "spec_decoding": true, "text_only": true, "encoder_parallel": false } }, "args": [ "--data-parallel-size", "2", "--attention-config", "{\"backend\":\"FLASHINFER_MLA_SPARSE_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}", "--kernel-config", "{\"enable_flashinfer_autotune\":true,\"moe_backend\":\"deep_gemm_mega_moe\"}", "--engram-config", "{\"cpu_offload\":true,\"embedding_across_dp\":true}", "--gpu-memory-utilization", "0.97", "--max-num-batched-tokens", "4096" ], "env": { "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "7200" }, "json_args": { "--speculative-config": { "enable_adaptive_verification": false } }, "points": { "dep2_c8": { "label": "8 (DEP2)", "concurrency": 8, "args": [ "--max-num-seqs", "16", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576]}", "--max-cudagraph-capture-size", "576" ] }, "dep2_c16": { "label": "16 (DEP2)", "concurrency": 16, "args": [ "--max-num-seqs", "16", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576]}", "--max-cudagraph-capture-size", "576" ] }, "dep2_c24": { "label": "24 (DEP2)", "concurrency": 24, "args": [ "--max-num-seqs", "32", "--prefill-schedule-interval", "64", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,640,768,896,1024,1280,1536,1792,2048,2304,2560,2816,3072,3328,3584,3840,4096]}", "--max-cudagraph-capture-size", "4096" ] }, "dep2_c32": { "label": "32 (DEP2)", "concurrency": 32, "default": true, "args": [ "--max-num-seqs", "32", "--prefill-schedule-interval", "64", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,640,768,896,1024,1280,1536,1792,2048,2304,2560,2816,3072,3328,3584,3840,4096]}", "--max-cudagraph-capture-size", "4096" ] }, "dep2_kv_c64": { "label": "64 (DEP2 + host KV)", "concurrency": 64, "args": [ "--max-num-seqs", "48", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576]}", "--max-cudagraph-capture-size", "576", "--gpu-memory-utilization", "0.955", "--kv-offloading-backend", "native", "--kv-offloading-size", "32" ] }, "dep2_kv_c128": { "label": "128 (DEP2 + host KV)", "concurrency": 128, "args": [ "--max-num-seqs", "48", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576]}", "--max-cudagraph-capture-size", "576", "--gpu-memory-utilization", "0.955", "--kv-offloading-backend", "native", "--kv-offloading-size", "32" ] }, "dep4_c96": { "label": "96 (DEP4)", "concurrency": 96, "args": [ "--data-parallel-size", "4", "--max-num-seqs", "96", "--max-num-batched-tokens", "8192", "--prefill-schedule-interval", "48", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048,2304,2560,2816,3072,3328,3584,3840,4096]}", "--max-cudagraph-capture-size", "8190" ] }, "dep4_c128": { "label": "128 (DEP4)", "concurrency": 128, "args": [ "--data-parallel-size", "4", "--max-num-seqs", "128", "--max-num-batched-tokens", "8192", "--prefill-schedule-interval", "48", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048,2304,2560,2816,3072,3328,3584,3840,4096]}", "--max-cudagraph-capture-size", "8190" ] } }, "runbook": "The host-KV points add vLLM's native CPU KV offloading, 32 GiB per DP rank (64 GiB per\nserver), and lower GPU memory to 0.955 to keep headroom for the indexer logits; with\n48 sequences per rank, about 32 of 128 sessions queue.\n\nThe lane fronts the server with vLLM Router 0.1.14 (`--policy consistent_hash`,\n`--intra-node-data-parallel-size` equal to the server's DP size, as on the router tab),\nplus `--request-timeout-secs 14400` so long agentic requests outlive the 1800 s default\nand `--disable-retries` so a failed request is not replayed.\n", "json": "/deepseek-ai/DeepSeek-V4.1-Flash/presets/b200_agentx_dep.json" }, "b300_agentx_tp": { "label": "AgentX TP4", "hardware": "b300", "strategy": "single_node_tp", "variants": [ "fp8", "default" ], "workload": "Agentic (InferenceX AgentX)", "description": "TP4 for concurrency 1 to 16 on the specialized decode image: latency-specialized decode steps, THP-backed Engram tables and FlashInfer autotuning.", "source": { "label": "InferenceX #4332", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4332", "key": "dsv41flash-fp4-b300-vllm-agentic-dspark", "revision": "649dd5a" }, "image": "vllm/vllm-openai:cu134-nightly-x86_64-dev-20261019-mk@sha256:2b9fc4b18d0aaabe4d5cad020d9270106472ebe65f5d3bc02c50526b8d9fd280", "select": { "frontend": "rust", "features": { "spec_decoding": true, "text_only": true, "encoder_parallel": false } }, "tp": 4, "args": [ "--kernel-config", "{\"enable_flashinfer_autotune\":true}", "--engram-config", "{\"cpu_offload\":true,\"use_thp\":true}", "--additional-config", "{\"decode_backend\":\"latency_specialized\"}" ], "env": { "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "7200" }, "json_args": { "--speculative-config": { "enable_adaptive_verification": false } }, "points": { "c1": { "label": "1", "concurrency": 1, "args": [ "--max-num-seqs", "1", "--max-num-batched-tokens", "4096", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[5,6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092]}", "--max-cudagraph-capture-size", "4092" ] }, "c8": { "label": "8", "concurrency": 8, "default": true, "args": [ "--max-num-seqs", "16", "--max-num-batched-tokens", "8192", "--prefill-schedule-interval", "160", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[5,6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}", "--max-cudagraph-capture-size", "8190" ] }, "c16": { "label": "16", "concurrency": 16, "args": [ "--max-num-seqs", "32", "--max-num-batched-tokens", "8192", "--prefill-schedule-interval", "128", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[5,6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048,2304,2560,2816,3072,3328,3584,3840,4096]}", "--max-cudagraph-capture-size", "8190" ] } }, "runbook": "The image carries the specialized decode from\n[vllm-project/vllm#61868](https://github.com/vllm-project/vllm/pull/61868) (on main\nsince 2026-10-21) and its DSpark draft path from\n[vllm-project/vllm#61869](https://github.com/vllm-project/vllm/pull/61869) (open), so\n`decode_backend` needs this image or a build carrying both.\n", "json": "/deepseek-ai/DeepSeek-V4.1-Flash/presets/b300_agentx_tp.json" }, "b300_agentx_dep": { "label": "AgentX DEP2", "hardware": "b300", "strategy": "single_node_dep", "variants": [ "fp8", "default" ], "workload": "Agentic (InferenceX AgentX)", "description": "DEP2 for concurrency 8 to 64 with FlashInfer sparse MLA, and with MegaAttention and lazy host-KV offload at 128 and 192, behind a consistent-hash vLLM Router.", "source": { "label": "InferenceX #4332", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4332", "key": "dsv41flash-fp4-b300-vllm-agentic-dspark", "revision": "649dd5a" }, "image": "vllm/vllm-openai:cu134-nightly-x86_64-dev-20261019-mk@sha256:2b9fc4b18d0aaabe4d5cad020d9270106472ebe65f5d3bc02c50526b8d9fd280", "select": { "frontend": "rust", "features": { "spec_decoding": true, "text_only": true, "encoder_parallel": false } }, "args": [ "--data-parallel-size", "2", "--kernel-config", "{\"enable_flashinfer_autotune\":true,\"moe_backend\":\"deep_gemm_mega_moe\"}", "--engram-config", "{\"cpu_offload\":true,\"embedding_across_dp\":true}", "--attention-config", "{\"backend\":\"FLASHINFER_MLA_SPARSE_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}", "--max-cudagraph-capture-size", "8190" ], "env": { "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "7200" }, "json_args": { "--speculative-config": { "enable_adaptive_verification": false } }, "points": { "c8": { "label": "8", "concurrency": 8, "args": [ "--max-num-seqs", "16", "--max-num-batched-tokens", "16384", "--prefill-schedule-interval", "80", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}" ] }, "c16": { "label": "16", "concurrency": 16, "args": [ "--max-num-seqs", "16", "--max-num-batched-tokens", "16384", "--prefill-schedule-interval", "80", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190]}" ] }, "c32": { "label": "32", "concurrency": 32, "default": true, "args": [ "--max-num-seqs", "32", "--max-num-batched-tokens", "16384", "--prefill-schedule-interval", "80", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048,2304,2560,2816,3072,3328,3584,3840,4096]}" ] }, "c64": { "label": "64", "concurrency": 64, "args": [ "--max-num-seqs", "64", "--max-num-batched-tokens", "32768", "--prefill-schedule-interval", "56", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048,2304,2560,2816,3072,3328,3584,3840,4096]}" ] }, "kv_c128": { "label": "128 (host KV)", "concurrency": 128, "select": { "kv_offload": "simple" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": 53687091200, "lazy_offload": true } } }, "args": [ "--attention-config", "{\"backend\":\"FLASHMLA_MEGA_ATTN_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}", "--max-num-seqs", "128", "--max-num-batched-tokens", "16384", "--prefill-schedule-interval", "32", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092,6144,8190],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048,2304,2560,2816,3072,3328,3584,3840,4096]}" ] }, "kv_c192": { "label": "192 (host KV)", "concurrency": 192, "select": { "kv_offload": "simple" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "cpu_bytes_to_use_per_rank": 53687091200, "lazy_offload": true } } }, "args": [ "--attention-config", "{\"backend\":\"FLASHMLA_MEGA_ATTN_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}", "--max-num-seqs", "192", "--max-num-batched-tokens", "16384", "--prefill-schedule-interval", "32", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046,3072,4092],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048]}", "--max-cudagraph-capture-size", "4092" ] } }, "runbook": "Uses the specialized decode image of **AgentX TP4** (the DEP2 points don't use the\nspecialized decode). The host-KV points offload 50 GiB per DP rank through the\nSimple CPU offload connector.\n\nThe lane fronts the server with vLLM Router 0.1.14 (`--policy consistent_hash`,\n`--intra-node-data-parallel-size` equal to the server's DP size, as on the router tab),\nplus `--request-timeout-secs 14400` so long agentic requests outlive the 1800 s default\nand `--disable-retries` so a failed request is not replayed.\n", "json": "/deepseek-ai/DeepSeek-V4.1-Flash/presets/b300_agentx_dep.json" }, "mi355x_agentx": { "label": "AgentX TP2", "hardware": "mi355x", "strategy": "single_node_tp", "variants": [ "fp8", "default" ], "workload": "Agentic (InferenceX AgentX)", "description": "TP2 for concurrency 1 to 64 with Engram CPU offload, and TP4 at 1 and 2 with resident Engram tables; per-point sequence caps, prefill budgets and graph sizes tuned for AgentX interactivity.", "source": { "label": "InferenceX #4480", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4480", "key": "dsv41flash-fp4-mi355x-vllm-agentic-dspark", "revision": "8ea833ccd" }, "image": "vllm/vllm-openai-rocm:nightly-e68efc685ab1e6f1eda004ced7157e3ca411da2f@sha256:520e098f54ec96837228aaddea0a833cbf6374dbdffd480c1d65f26ffab7701b", "select": { "frontend": "rust", "features": { "spec_decoding": true, "text_only": true, "encoder_parallel": false } }, "args": [ "--max-num-batched-tokens", "16384", "--max-cudagraph-capture-size", "1024" ], "env": { "OMP_NUM_THREADS": "1" }, "points": { "tp2_c1": { "label": "1 (TP2)", "concurrency": 1, "args": [ "--max-num-seqs", "1", "--max-num-batched-tokens", "32768", "--max-cudagraph-capture-size", "8" ] }, "tp2_c2": { "label": "2 (TP2)", "concurrency": 2, "args": [ "--max-num-seqs", "1", "--max-num-batched-tokens", "32768", "--max-cudagraph-capture-size", "8" ] }, "tp2_c4": { "label": "4 (TP2)", "concurrency": 4, "args": [ "--max-num-seqs", "1", "--max-num-batched-tokens", "32768", "--max-cudagraph-capture-size", "8" ] }, "tp2_c8": { "label": "8 (TP2)", "concurrency": 8, "default": true, "args": [ "--max-num-seqs", "2", "--max-num-batched-tokens", "8192", "--max-cudagraph-capture-size", "16" ] }, "tp2_c16": { "label": "16 (TP2)", "concurrency": 16, "args": [ "--max-num-batched-tokens", "8192", "--max-num-seqs", "4" ] }, "tp2_c32": { "label": "32 (TP2)", "concurrency": 32, "args": [ "--max-num-batched-tokens", "8192", "--max-num-seqs", "8" ] }, "tp2_c64": { "label": "64 (TP2)", "concurrency": 64 }, "tp4_c1": { "label": "1 (TP4)", "concurrency": 1, "description": "The Engram tables stay in HBM at TP4.", "tp": 4, "args": [ "--engram-config", "{\"cpu_offload\":false}", "--max-num-seqs", "1", "--max-num-batched-tokens", "32768", "--max-cudagraph-capture-size", "8" ] }, "tp4_c2": { "label": "2 (TP4)", "concurrency": 2, "description": "The Engram tables stay in HBM at TP4.", "tp": 4, "args": [ "--engram-config", "{\"cpu_offload\":false}", "--max-num-seqs", "1", "--max-num-batched-tokens", "32768", "--max-cudagraph-capture-size", "8" ] } }, "runbook": "Excess sessions queue behind the sequence cap. These settings target AgentX p90\ninteractivity; assess queueing, TTFT, end-to-end latency and throughput for your\ntraffic. The c16/c32 limits from [#4409](https://github.com/SemiAnalysisAI/InferenceX/pull/4409)\nand the 1024-token graph cap at c16/c32/c64 are retained.\n\n[InferenceX run 40161856449, attempt 3](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/40161856449/attempts/3)\npassed all nine performance points and its TP2 c64 accuracy job at `7cd3fae`.\nThe serving configuration is unchanged at audited head `8ea833ccd`; this is not\nan exact-head rerun or a claim of an independent accuracy job for every point.\nThroughput runs use DSpark's synthetic golden acceptance length (3.51); the\npreset keeps real block rejection for ordinary serving and accuracy evaluation.\n", "json": "/deepseek-ai/DeepSeek-V4.1-Flash/presets/mi355x_agentx.json" }, "gb300_agentx_pd": { "label": "AgentX PD", "hardware": "gb300", "strategy": "pd_cluster", "variants": [ "fp8", "default" ], "workload": "Agentic (InferenceX AgentX)", "default": true, "description": "The lane's engine settings in a 1P1D layout: one tray of DEP4 prefill (the lane runs several DEP2 prefill engines behind Dynamo) and four trays of DEP16 decode, with MegaMoE, MegaAttention and Engram offload; KV moves over the NVLink fabric.", "source": { "label": "InferenceX #3941", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/3941", "key": "dsv41flash-fp4-gb300-dynamo-vllm-agentic-dspark-disagg", "revision": "f516173" }, "args": [ "--kv-cache-dtype", "fp8", "--attention-config", "{\"backend\":\"FLASHMLA_MEGA_ATTN_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}", "--kernel-config", "{\"moe_backend\":\"deep_gemm_mega_moe\",\"sparse_indexer_topk_backend\":\"deep_select\"}", "--engram-config", "{\"cpu_offload\":true}", "--enable-ep-weight-filter", "--gpu-memory-utilization", "0.95", "--numa-bind" ], "env": { "VLLM_USE_V2_MODEL_RUNNER": "1", "UCX_TLS": "cuda_copy,cuda_ipc,tcp", "UCX_CUDA_IPC_ENABLE_MNNVL": "y" }, "roles": { "prefill": { "parallelism": "dep", "args": [ "--max-num-seqs", "64", "--max-num-batched-tokens", "32768", "--long-prefill-token-threshold", "8192", "--compilation-config", "{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046],\"max_cudagraph_capture_size\":2046}" ] }, "decode": { "nodes": 4, "parallelism": "dep", "args": [ "--max-num-seqs", "40", "--max-num-batched-tokens", "400", "--compilation-config", "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96,102,108,114,120,126,132,138,144,150,156,162,168,174,180,186,192,198,204,210,216,222,228,234,240],\"max_cudagraph_capture_size\":240}", "--no-enable-prefix-caching" ] } }, "json": "/deepseek-ai/DeepSeek-V4.1-Flash/presets/gb300_agentx_pd.json" } }, "hardware_notes": { "dgx_station_gb300": { "title": "Weights in Grace memory (UVA offload)", "body": "Single GB300 with both Engram tables and 60 GiB of routed experts in pinned Grace memory (UVA offload). Measured on a nightly image; vLLM 0.31.0 carries every flag this row uses. The first boot on a new flag set runs a 75-90 min FlashInfer autotune; keep /root/.cache/vllm on a persistent volume." } }, "guide": "## Overview\n\nDeepSeek-V4.1 is a vision-language Mixture-of-Experts model: **552B backbone parameters\nplus 196B of Engram memory, 8B active per prompt token and 16B per output token**, 40\ntransformer layers at hidden size 5120, with a 32-layer ViT and aligner in front of the\ntext stack. Six things distinguish it from V4:\n\n- **Causal encoder-decoder.** The 40 layers split into a 20-layer encoder and a\n 20-layer decoder, and the decoder's global KV is projected from the encoder's final\n hidden state instead of being computed layer by layer. Prefill therefore only runs\n the encoder half, which is where the 8B-prefill / 16B-decode split comes from.\n- **Two-tier sparse attention.** Every layer attends over a 128-token sliding window.\n Layers that carry a compression ratio add compressed KV latents reaching further\n back, pooled by a learned softmax gate. A small side attention — the *indexer*,\n inherited from V3.2-Exp — scores those latents and keeps the best 512 per query,\n pre-filtered by a candidate stage that selects 2048 blocks of 8. Only four layers\n (2, 8, 14, 20) actually compress their own KV; the rest read that cache. V4.1 uses\n only compression ratios 1 and 2, where V4 used 4 and 128.\n- **Engram n-gram memory.** Layers 1 and 14 each own a hash table of ~384M rows x 256\n dims, looked up by 2-, 3- and 4-gram hashes of the input and written into the residual\n stream through a learned gate. These two tables alone are **196.6B parameters (~183 GiB)** —\n plan capacity for them, they dominate everything except the experts.\n- **Hyper-Connections.** The residual stream is carried as 4 parallel copies; each\n sublayer derives its own pre/post/combine coefficients from the stream, with the\n combine matrix made doubly stochastic by 20 Sinkhorn iterations.\n- **DSpark draft head.** Three stages (128 routed experts each, 3 activated) draft a\n block of 5 tokens, reading the attention input of layers 37-39, with a Markov bias\n head and a confidence head on the last stage.\n- **Mixed MXFP4/MXFP8 checkpoint.** Routed expert weights are MXFP4; everything else is\n MXFP8 block-quantized, UE8M0 scales throughout. Embedding and LM head are BF16.\n\nRouting is 6 of 384 experts per token plus one shared expert, scored with\n`sqrtsoftplus` and a `noaux_tc` bias — with a **separate routing bias for tokens\ninside an image span**, so vision and text tokens do not compete for the same experts.\n\n## Checkpoints\n\n| Variant | Checkpoint | Notes |\n|---|---|---|\n| **FP8 (1017)** (default) | [deepseek-ai/DeepSeek-V4.1-Flash-1017](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017) | 2026-10-17 weight update — recommended |\n| **FP8 (0909)** | [deepseek-ai/DeepSeek-V4.1-Flash](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash) | Launch release (2026-09-09) |\n\n**1017 is the default here.** It is a minor update with the same architecture,\n`config.json`, tokenizer and prompt encoding as the launch checkpoint — only the\nweights change. DeepSeek reports gains across reasoning, coding, agentic and visual\nbenchmarks (e.g. Terminal-Bench 4.0 31.2 → 42.3, HLE w/o tools 36.8 → 42.6) along with\nshorter outputs. Both serve with the same command; the Variant row only swaps the model\nid. Like the launch repo it ships no Jinja chat template, which the\n`--tokenizer-mode deepseek_v41` flag already covers. Pick **FP8 (0909)** to reproduce\nearlier results.\n\nThe verified hardware badges and the measured numbers in the sections below\n(benchmarks, KV offloading validation) were taken on the 0909 launch checkpoint; the\n1017 weights share its config and footprint but have not been separately re-run.\n\n## Context length\n\n1,048,576 tokens, reached by YaRN with factor 16 over a 65,536-token training window.\nCompressed KV rotates at its own RoPE theta (160,000) because one latent stands for\nseveral tokens, so its positions are further apart than the raw stream's.\n\n## Images\n\nImages enter the prompt as `<|deepseek_image|>` spans; every position in the span\ncarries the image token id, and the per-position role (start / newline / end / image)\ncomes from the processor. The vision tower is a 32-layer ViT at hidden size 1024, patch\n14, with a 3x downsampling aligner and a cap of 1024 tokens per image (minimum 295,936\npixels). There is **no limit on images per prompt**. Merged embeddings enter the text\nmodel as `inputs_embeds`, before the hyper-connection stream expansion, while raw token\nids still flow through so the router can apply the image routing bias.\n\nTick **Encoder parallel** to run the ViT data-parallel (`--mm-encoder-tp-mode data`)\ninstead of tensor-parallel: at 32 layers / hidden 1024 the encoder is small enough\nthat TP communication costs more than it saves, which can significantly reduce TTFT\nfor multi-image requests. It is mutually exclusive with **Text only**.\n\n## Reasoning and tool calling\n\nTwo thinking modes and a **numeric reasoning budget** rather than discrete tiers.\nSend them as `chat_template_kwargs`:\n\n| key | values |\n|---|---|\n| `thinking` / `enable_thinking` | boolean; if you send both they must agree |\n| `reasoning_effort` | `low` (25), `high` (50), `xhigh` (75), `max` (100), **or an integer 1-100** |\n\nThe top-level OpenAI `reasoning_effort` field also works, where `\"none\"` turns\nthinking off. `\"minimal\"` and `\"medium\"` are rejected — they are not part of this\nmodel's set.\n\nNote that the DeepSeek API maps its `low` / `high` / `max` tiers to 50 / 75 / 100,\nwhile the open-source prompt encoder (and therefore vLLM) maps `low` / `high` /\n`xhigh` / `max` to 25 / 50 / 75 / 100. A request tuned at `high` on the API runs at\neffort 50 here; pass an integer if you need parity.\n\n> **With both keys unset, thinking is ON at effort 50.** The do-nothing config is\n> the most verbose one, so a request with a small `max_tokens` spends its budget on\n> the trace and returns empty `content` with `finish_reason=length`. That reads as a\n> broken model and is not. Either pin `thinking: false` or give the budget room.\n\nIn thinking mode the budget is rendered into the prompt as a `Reasoning Effort: N`\nprefix on the first turn only.\n\n```python\nresp = client.chat.completions.create(\n model=\"deepseek-ai/DeepSeek-V4.1-Flash-1017\",\n messages=[{\"role\": \"user\", \"content\": \"What is 17*19?\"}],\n extra_body={\"chat_template_kwargs\": {\"thinking\": True, \"reasoning_effort\": 25}},\n)\n```\n\nDeepSeek's published numbers use `temperature=1.0`, `top_p=0.95` and effort 100 with\n`max_tokens` of at least 256K. The tech report notes that efforts of 60-80 recover most\nof that accuracy at well under half the tokens, so reserve `max` for hard tasks.\n\nTool calls are wrapped in DSML tag blocks rather than JSON fences, and tool output\ncomes back in `` tags.\n\n## Speculative decoding\n\nDSpark is the only speculative method for this checkpoint. V4.1 dropped the MTP module\nthat V3 and V4 trained alongside the backbone. **Speculative decoding** enables a 5-token\nblock. On NVIDIA it also turns on adaptive verification (see the feature description).\nOn AMD, vLLM currently refuses adaptive verification. The generated command sets\n`enable_adaptive_verification:false` and verifies the full block. Two checks in\n`maybe_create_adaptive_verification_manager` fail on ROCm: the indexer helper\n`supports_device_cpu_query_lens_mismatch()` is False, and\n`DeepseekV41ROCMAiterSparseSWABackend` reports `AttentionCGSupport.UNIFORM_BATCH`\nrather than `ALWAYS`. Remove the AMD override only after a serve with the true flag\nstarts. Acceptance depends on the workload, so measure it on your own traffic before\nsizing a deployment around it.\nThe drafter's experts add roughly 14B parameters to the load.\n\n## Serving text-only\n\nTick **Text only** to add `--language-model-only`, which skips the vision encoder\nentirely. Worth it whenever the workload is text: it drops the ViT and aligner from\nthe load and frees that VRAM for KV cache. It is mutually exclusive with\n`encoder_parallel`. The verified GB200 runs (TP4 and 1P1D) were text-only.\n\n## Prefill/Decode disaggregation\n\nThe **Prefill/Decode Disaggregation** strategy is the verified 1P1D layout on GB200\nNVL4: one tray (4 GPUs) per role, TP4 in each pool, KV handed over through NIXL,\nfronted by `vllm-router --vllm-pd-disaggregation`. Both pools disable FlashInfer\nautotune plus JIT and CuTeDSL warmup via `--kernel-config`, skip the DeepGEMM warmup\n(`VLLM_DEEP_GEMM_WARMUP=skip`), and cap `--max-num-seqs` at 32. On 8-GPU nodes the\nsame layout becomes TP8 per role.\n\nWith **Speculative decoding** on, DSpark runs in both pools so the transferred KV\nstays compatible.\n\n### GB300\n\nGB300's default **AgentX PD** preset runs both pools as DEP, per\n[SemiAnalysisAI/InferenceX#3941](https://github.com/SemiAnalysisAI/InferenceX/pull/3941):\none tray of DEP4 prefill, four trays of DEP16 decode. Keep them on DEP; MegaMoE needs\nexpert parallelism.\n\nTo scale prefill, InferenceX ran two DEP2 engines per tray. Drop the\n`--data-parallel-{start-rank,address,rpc-port}` flags from the prefill command and\nrun it twice:\n\n```bash\nCUDA_VISIBLE_DEVICES=0,1 VLLM_NIXL_SIDE_CHANNEL_PORT=5557 vllm serve ... --port 8001 \\\n --data-parallel-size 2 --data-parallel-size-local 2\nCUDA_VISIBLE_DEVICES=2,3 VLLM_NIXL_SIDE_CHANNEL_PORT=5559 vllm serve ... --port 8011 \\\n --data-parallel-size 2 --data-parallel-size-local 2\n# vllm-router: list both ports, with --intra-node-data-parallel-size 1\n```\n\n## Memory\n\nThe checkpoint is roughly **511 GB on disk** (476 GiB), which breaks down as:\n\n| Component | Entries | Stored |\n|---|---:|---:|\n| Routed + DSpark experts (MXFP4) | 557.2B | 259.5 GiB |\n| Engram tables (FP8) | 196.6B | 183.1 GiB |\n| Attention, dense projections, routers (FP8) | 7.4B | 6.9 GiB |\n| Embedding + LM head (BF16), norms (FP32) | 2.0B | 3.9 GiB |\n| UE8M0 block scales | 23.6B | 21.9 GiB |\n\nDeepSeek's 552B backbone figure covers the routed experts, attention and embeddings;\nthe Engram tables, the ~14B DSpark drafter and the block scales sit outside it.\n\n`vram_minimum_gb: 614` is that total times the schema's 1.2 headroom factor. It fits\none GB200 NVL4 tray (768 GB) at TP4, or one 8-GPU H200 node (1128 GB) with room for\nKV cache.\n\nKV cache is a small share of that budget. The compressed latents are shared across\nlayers and trained to be stored in FP4, and DeepSeek puts the resulting global KV at\n**890 bytes per token**, about a quarter of V4-Flash; by that accounting a full 1M-token\nprompt holds under 1 GB of global KV, plus a fixed 128-token sliding window per layer.\nWeights and batch size, not cache, set the capacity limit — still, measure before\nassuming a given context/batch fits.\n\n## Prerequisites\n\n- vLLM 0.31.0+, from the pip wheel or the release image. The architecture shipped in\n 0.30.0 ([vllm-project/vllm#56228](https://github.com/vllm-project/vllm/pull/56228));\n 0.31.0 adds the KV-offload fix ([vllm-project/vllm#57145](https://github.com/vllm-project/vllm/pull/57145))\n and the ROCm Engram host-memory gate ([vllm-project/vllm#57491](https://github.com/vllm-project/vllm/pull/57491)).\n The launch-day `deepseekv41-flash-0909` image is no longer needed.\n- On AMD the Docker tab uses `vllm/vllm-openai-rocm:nightly-rocm100`, the ROCm 10.0\n nightly the AMD rows were verified on. **MI355X and MI325X need it**: their\n commands pass `--monokernel`, and MI355X also passes the ROCm MXFP4 indexer.\n Both are on `main` but not in 0.31.0, so the Install block offers only Docker\n on those two SKUs. MI300X and MI350X also run from the ROCm pip wheel.\n- The Rust OpenAI frontend is selected by default in the command builder. Switch\n to Python if you encounter unsupported features or compatibility issues.\n- Expect a long first load: `VLLM_ENGINE_READY_TIMEOUT_S=3600` is set for that reason.\n- On AMD, the generated command sets `VLLM_USE_BREAKABLE_CUDAGRAPH=1`. DeepSeek-V4.1-Flash does not support `torch.compile`, and the ROCm sparse SWA backend only supports uniform-batch CUDA graphs. Without breakable CUDA graphs, default `FULL_AND_PIECEWISE` dies at capture.\n- On AMD, the generated command passes `--moe-backend aiter` rather than naming a kernel. Naming `aiter_triton_mxfp4_bf16` pins the Triton W4A16 `_moe_gemm_a16w4` kernel; the plain name lets vLLM select the Composable Kernel a8w4 experts, used in the MI355X benchmark below. It also sets `AITER_TRITON_LOG_LEVEL=ERROR` to drop AITER's per-call \"Gluon unavailable\" warning, which is constant on gfx950 and dominated the server log.\n- On MI355X, **Tensor Parallel** inherits vLLM's scheduler limits. Tune `--max-num-seqs` for your traffic and memory budget if needed. The benchmark configuration below records an explicit limit for reproducibility.\n- On AMD, ticking **Speculative decoding** also sets `enable_adaptive_verification:false`. vLLM currently refuses the true flag: `DeepseekV41IndexerBackend.supports_device_cpu_query_lens_mismatch()` is False, and `DeepseekV41ROCMAiterSparseSWABackend` reports `UNIFORM_BATCH` rather than `ALWAYS`. DSpark still drafts 5 tokens per round.\n\n## Verifying\n\nServe, then send one text request and one image request — the vision tower is a\nseparate path and a text-only smoke test will not exercise it:\n\n```bash\ncurl http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\":\"deepseek-ai/DeepSeek-V4.1-Flash-1017\",\n \"messages\":[{\"role\":\"user\",\"content\":\"What is 17*19? Return only the integer.\"}]}'\n```\n\nA correct answer is `323`.\n\n## Performance tuning\n\nStart with vLLM's defaults and tune one parameter at a time for your workload:\n\n- `--max-num-seqs`: raise the running-sequence limit if it causes queueing and\n memory permits; lower it if larger batches hurt latency or memory use.\n- `--max-num-batched-tokens`: balance prefill throughput against decode latency\n and memory use by adjusting the token budget per scheduling iteration.\n- `--max-cudagraph-capture-size`: cover expected decode batches, including DSpark\n draft tokens, while accounting for graph memory and startup cost: 128 sequences\n with five drafts need `128 * (1 + 5) = 768` tokens, which the MI355X AgentX\n preset rounds up to 1024.\n- `--max-model-len`: size the context limit for your prompt plus output lengths.\n\n## MI355X Tensor Parallel\n\nOn MI355X, **Tensor Parallel** defaults to **TP2** with Engram CPU offload\n(`--engram-config '{\"cpu_offload\":true}'`). The Engram tables cost 47.2 GiB of\ndevice memory per rank at TP4 and 94.4 GiB at TP2; on a 288 GiB card the TP2\nfigure does not fit beside half the checkpoint, so the tables move to pinned\nhost memory. TP2 halves the GPUs per server and so doubles the servers per\nnode. For TP4, pass `--tensor-parallel-size 4` and\n`--engram-config '{\"cpu_offload\":false}'` to keep the tables resident.\n\nThe offload resolves on gfx950 only on builds carrying\n[vllm-project/vllm#57491](https://github.com/vllm-project/vllm/pull/57491),\nwhich widened two `is_cuda()` gates to `is_cuda_alike()`. Those builds also\ndefault `cpu_offload` on through `VLLM_PLE_CPU_OFFLOAD`, so set it explicitly\nrather than inheriting the default.\n\n`--monokernel` selects the mono decode layer from\n[vllm-project/vllm#60397](https://github.com/vllm-project/vllm/pull/60397),\nwhich [vllm-project/vllm#61082](https://github.com/vllm-project/vllm/pull/61082)\nextended to all 40 backbone layers and moved from the\n`VLLM_ROCM_MONO_DECODE` environment variable to this flag. Two persistent\nlaunches cover a layer for decode steps of up to 48 rows, which with five\nDSpark drafts is concurrency 8; larger steps fall back to the regular path, so\nthe gain is largest at low concurrency and fades as concurrency rises.\n\nDecoder SWA bounded replay\n([vllm-project/vllm#59039](https://github.com/vllm-project/vllm/pull/59039))\nis on by default on ROCm, so the recipe no longer passes\n`--no-swa-bounded-replay`. Use a nightly from 2026-10-13 or later; earlier\nROCm builds lack the replay window clamp and fault at the first prefix hit.\n\n### AgentX on MI355X\n\nThe **AgentX TP2** preset (opt-in;\n[InferenceX #4480](https://github.com/SemiAnalysisAI/InferenceX/pull/4480)) renders the\nlane on its pinned ROCm image: TP2 for concurrency 1 to 64 with Engram CPU offload,\nand TP4 at concurrency 1 and 2 with resident Engram tables. Its points carry the\ncorresponding sequence caps, prefill budgets and graph sizes. The lane served\n**FP8 (0909)**. These limits are workload-specific, not generic defaults.\n\n## MI300X Tensor Parallel\n\nOn MI300X, **Tensor Parallel** takes the recipe default of **TP4** with the\nEngram tables offloaded to pinned host memory\n(`--engram-config '{\"cpu_offload\":true}'`). At TP4 the resident weights are\nroughly 81 GiB per rank, which leaves a healthy KV pool on a 192 GB card.\n\n**TP2 does not fit on this SKU.** At TP2 the offloaded weights rise to roughly\n145 GiB per rank, and a measured run reported\n`Available KV cache memory: -13.51 GiB`, so the engine refused to start with\n`No available memory for the cache blocks`. That was already with the\nsparse-attention indexer buffer halved to 4096 batched tokens. Raising\n`--gpu-memory-utilization` and halving the buffer again recover roughly 13 GiB\nbetween them, which still leaves no usable KV for a 1M-context model, so TP2\nis not offered here. MI325X runs the same TP2 configuration successfully on\nits larger 256 GB card.\n\nMI300X is gfx942 and needs the same two workarounds as MI325X: no FP4 MFMA, so\nthe MXFP4 experts run through a Triton kernel and device memory is better spent\non KV than on resident Engram tables; and piecewise graph capture segfaults\nevery worker, so this recipe sets `VLLM_USE_BREAKABLE_CUDAGRAPH=0` and captures\nonly full decode graphs\n(`--compilation-config '{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}'`), with prefill\neager.\n\nThis recipe does not pass `--monokernel`. With the kernels' weight copies\nincluded, #60731 measured 170.11 GiB of weights a rank. At\n`--gpu-memory-utilization 0.9` a 192 GiB MI300X leaves about 2.7 GiB beside\nthose weights.\n\nThe offload resolves on gfx942 only on builds carrying\n[vllm-project/vllm#57491](https://github.com/vllm-project/vllm/pull/57491),\nthe same gate MI325X and MI355X need. Those builds also default `cpu_offload`\non through `VLLM_PLE_CPU_OFFLOAD`, so set it explicitly rather than inheriting\nthe default.\n\n## MI325X Tensor Parallel\n\nOn MI325X, **Tensor Parallel** takes the recipe default of **TP4**, with the\nEngram tables offloaded to pinned host memory\n(`--engram-config '{\"cpu_offload\":true}'`). A 256 GB card leaves roughly\n81 GiB per rank of resident weights at TP4 and roughly 145 GiB at TP2; TP2\nfits but leaves the smallest KV pool of any arm in this recipe, so TP4 is the\ndefault here rather than the TP2 that MI355X uses on its larger 288 GiB card.\nFor TP2, pass `--tensor-parallel-size 2`.\n\nMI325X is gfx942, which differs from gfx950 in two ways that matter. It has no\nFP4 MFMA, so the MXFP4 experts run through a Triton kernel and device memory is\nbetter spent on KV than on resident Engram tables. It also segfaults in every\nworker during piecewise graph capture, so this recipe sets\n`VLLM_USE_BREAKABLE_CUDAGRAPH=0` and captures only full decode graphs\n(`--compilation-config '{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}'`); prefill runs\neagerly.\n\n`--monokernel` selects the gfx942 mono decode kernels from\n[vllm-project/vllm#60731](https://github.com/vllm-project/vllm/pull/60731).\nThey run at TP4 only, on decode steps of up to 42 rows (7 requests with\nfive DSpark drafts). Larger steps, prefill, and mixed batches take the\nregular path, and TP2 stays on vLLM's path. With their weight copies\nincluded, #60731 measured 170.11 GiB of weights a rank, up from 98.88 GiB.\nAt `--gpu-memory-utilization 0.9` the KV pool still held 11.18 million\ntokens. Use a ROCm nightly from 2026-10-20 or later;\n`nightly-rocm100-df3bd094aeea829f27baa0a5282830464f601d9f` is the first.\nEarlier builds have the flag for gfx950 only and refuse to start on MI325X.\n\nThe offload resolves on gfx942 only on builds carrying\n[vllm-project/vllm#57491](https://github.com/vllm-project/vllm/pull/57491),\nthe same gate MI355X needs. Those builds also default `cpu_offload` on through\n`VLLM_PLE_CPU_OFFLOAD`, so set it explicitly rather than inheriting the\ndefault.\n\n## H200 Tensor Parallel\n\nOn H200, **Tensor Parallel** uses the TP4 default with Engram CPU offload\n(`--engram-config '{\"cpu_offload\":true}'`). The Engram tables move to pinned\nhost DRAM and are read through UVA, so the output is unchanged. On 141 GB\ncards that offload is what makes TP4 practical: it moves 23.6 GiB per rank off\nthe GPU, leaving 81.2 GiB of resident weights and 38.5 GiB of KV per GPU\n(about 20M tokens). No scheduler caps are needed at this replica size.\n\n## Blackwell TP and DEP\n\nOn B200, B300, GB200 and GB300, **Tensor Parallel** uses **TP2** with\n`FLASHINFER_MLA_SPARSE_DSV41` and Engram CPU offload\n(`--engram-config '{\"cpu_offload\":true}'`), which moves the Engram tables to\npinned host DRAM and reads them through UVA; the output is unchanged.\n**Data + Expert Parallel** uses the hardware's\nGPU count: DEP4 on GB200/GB300 trays, DEP8 on B200/B300 nodes. It enables\n`--enable-expert-parallel`, `FLASHMLA_MEGA_ATTN_DSV41`,\n`--kernel-config '{\"moe_backend\":\"deep_gemm_mega_moe\"}'`, and\n`--engram-config '{\"embedding_across_dp\":true}'`.\n\nBoth configurations set `indexer_kv_dtype` to `mxfp4`, `indexer_sparse_logits`\nto `true`, `--kv-cache-dtype fp8`, and `--max-num-seqs 128`. Reasoning uses\n`--reasoning-parser deepseek_v41`.\n\nEach Blackwell SKU carries an exact-GPU override that takes precedence over\nthe shared Blackwell defaults. B300 keeps its existing CUDA Graph capture\nconfiguration (`--max-cudagraph-capture-size 8190`),\n`--max-num-batched-tokens 8192` and `--max-num-seqs 256`; B200, GB200 and\nGB300 add Engram CPU offload and otherwise inherit the shared Blackwell\nsettings. On B200 both strategies open on the InferenceX AgentX presets (see\n**AgentX tuning on NVIDIA GPUs**); pick **Baseline** in the Preset row for the\nshared Blackwell settings.\nScheduler limits are workload-dependent at TP2, so size\n`--max-num-batched-tokens` and `--max-cudagraph-capture-size` for your own\nserving profile. For high interactivity on any of them, use\n`--tensor-parallel-size 4`.\n\n## AgentX tuning on NVIDIA GPUs\n\nThe InferenceX AgentX lanes that run as one server are presets, each with the lane's\nconcurrency points (see the **Presets** tab):\n\n- **B200**: **AgentX TP** (TP4 for concurrency 1 to 128, TP2 at 1) and **AgentX DEP**\n (DEP2, DEP2 with a host KV tier, and DEP4), the defaults on B200\n ([InferenceX #3837](https://github.com/SemiAnalysisAI/InferenceX/pull/3837)).\n- **B300**: **AgentX TP4** and **AgentX DEP2** on the specialized decode image\n ([InferenceX #4332](https://github.com/SemiAnalysisAI/InferenceX/pull/4332)).\n\nThe lanes served the 0909 launch checkpoint, **FP8 (0909)**. The DEP presets use two or\nfour of the node's GPUs, so run one server per GPU group, each behind its own vLLM Router.\n\n### Dynamo disaggregated serving on B300 and GB300\n\nThe AgentX lanes in\n[InferenceX #4231](https://github.com/SemiAnalysisAI/InferenceX/pull/4231) (B300) and\n[InferenceX #4266](https://github.com/SemiAnalysisAI/InferenceX/pull/4266) (GB300) run\nN DEP2 prefill engines and one DEP decode engine, each with its own sidecar, behind one\nKV-routing frontend. Prefill pushes KV to decode with NIXL and keeps a host KV tier.\nThe `push_as_nixl` connector and `VLLM_MAX_ASYNC_KV_ADMITS_PER_STEP` come from the dev\nimages `vllm/vllm-openai:nightly-x86_64-dev-20261012-59e75f4` (B300) and\n`vllm/vllm-openai:cu134-nightly-aarch64-dev-20261019` (GB300).\n\n| GPU | Clients | Prefill engines | Decode | Decode `--max-num-seqs` | `VLLM_MAX_ASYNC_KV_ADMITS_PER_STEP` |\n|---|---|---:|---|---|---|\n| B300 | 256, 768, 1280, 2048, 3584 | 2, 2, 3, 4, 5 | DEP8 | 18, 40, 72, 132, 168 | 4 |\n| GB300 | 3018, 7388, 10888 | 6, 12, 16 | DEP32 | 18, 50, 76 | 2, 4, 5 |\n| GB300 | 9888 | 12 | DEP16 | 253 | 6 |\n\nBoth roles use TP1 with expert parallelism and these settings:\n\n```bash\nexport VLLM_USE_V2_MODEL_RUNNER=1 VLLM_ALLREDUCE_USE_SYMM_MEM=0 VLLM_RANDOMIZE_DP_DUMMY_INPUTS=1 NCCL_CUMEM_ENABLE=1\nexport UCX_MEMTYPE_CACHE=n UCX_MEMTYPE_REG_WHOLE=n UCX_RNDV_PIPELINE_ERROR_HANDLING=y\nexport VLLM_ENGINE_READY_TIMEOUT_S=7200 VLLM_RPC_TIMEOUT=600000\n--attention-config '{\"backend\":\"FLASHMLA_MEGA_ATTN_DSV41\",\"indexer_kv_dtype\":\"mxfp4\",\"indexer_sparse_logits\":true}'\n--kernel-config '{\"enable_flashinfer_autotune\":true,\"moe_backend\":\"deep_gemm_mega_moe\",\"sparse_indexer_topk_backend\":\"deep_select\"}'\n--speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"enable_adaptive_verification\":false}'\n--kv-cache-dtype fp8 --gpu-memory-utilization 0.95 --prefill-schedule-interval 2 --enable-ep-weight-filter --numa-bind\n```\n\n| | B300 | GB300 |\n|---|---|---|\n| NIXL backend | `LIBFABRIC` over EFA | `UCX` over MNNVL |\n| Env | `FI_PROVIDER=efa FI_EFA_USE_DEVICE_RDMA=1 FI_EFA_FORK_SAFE=1 UCX_TLS=cuda_copy,cuda_ipc,srd,tcp NIXL_UCX_SGL_ENABLE=0` | `UCX_TLS=cuda_copy,cuda_ipc,tcp UCX_CUDA_IPC_ENABLE_MNNVL=y NIXL_UCX_SGL_ENABLE=1 NCCL_MNNVL_ENABLE=1 NCCL_NVLS_ENABLE=1 VLLM_RS_TOKEN_IDS_AS_BYTES=1` |\n| Allocator | `--no-enable-cumem-allocator` (EFA cannot register VMM memory) | `--enable-cumem-allocator` |\n| DSpark | adds `\"rejection_sample_method\":\"block\"` | as above |\n| Engram | `{\"cpu_offload\":true,\"embedding_across_dp\":true,\"use_thp\":false}` | same; decode sets `\"cpu_offload\":false` |\n| Host KV per prefill rank | 70 GiB | 80 GiB |\n| Prefill `--max-num-batched-tokens` | 16384 | 32768 |\n| Decode EPLB redundant experts | 16 | 16 at DEP16, 32 at DEP32 |\n| Decode `--numa-bind-nodes` | `0 0 0 0 1 1 1 1` | `0 0 1 1` |\n\nOn B300, set `--cpu-distributed-timeout-seconds 7200` for the Engram load. NIXL's\nLIBFABRIC plugin also needs libfabric and rdma-core from the AWS EFA installer and the\nCUDA 12 runtime (`nvidia-cuda-runtime-cu12`), which the image lacks.\n\nPrefill runs `--data-parallel-size 2` per GPU pair with `--max-num-seqs 64`,\n`--long-prefill-token-threshold 8192` and `--prefix-cache-retention-interval 32768`. Its\nKV config, shown for B300 (GB300 uses `\"backends\":[\"UCX\"]` and `85899345920` bytes):\n\n```json\n{\"kv_connector\":\"MultiConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"connectors\":[\n {\"kv_connector\":\"NixlConnector\",\"kv_connector_module_path\":\"vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_as_nixl\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"backends\":[\"LIBFABRIC\"],\"enforce_handshake_compat\":false,\"kv_lease_duration\":120}},\n {\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":75161927680,\"lazy_offload\":true}}]}}\n```\n\nPrefill graphs:\n`{\"mode\":\"VLLM_COMPILE\",\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2046],\"decoder_replay_cudagraph_capture_sizes\":[6,12,18,24,30,36,48,60,72,96,120,144,192,240,288,384,480,576,768,1020,1536,2048],\"max_cudagraph_capture_size\":2046}`.\n\nDecode uses the `NixlConnector` entry alone with `\"kv_role\":\"kv_consumer\"` and\n`--no-enable-prefix-caching`. It sets `--max-num-batched-tokens` to ten times\n`--max-num-seqs` and captures FULL decode-only graphs in steps of 6 up to six times\n`--max-num-seqs`. It balances experts with `--enable-eplb` and\n`--eplb-config '{\"communicator\":\"symm_mem\",\"num_redundant_experts\":16,\"window_size\":1000,\"step_interval\":3000,\"use_async\":true}'`.\n\nLaunch the sidecars as in **Dynamo sidecar serving**, with\n`--grpc-startup-deadline-secs 3600`. Start the frontend so that every request prefills\non a prefill engine and each session stays on one prefill engine's prefix and host KV\ncache. On GB300, also set `DYN_ROUTER_DECODE_ACTIVE_REQUEST_WEIGHT=10000000`.\n\n```bash\nexport DYN_ROUTER_TEMPERATURE=0 DYN_ROUTER_QUEUE_THRESHOLD=None DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS=3600\nexport DYN_TOKENIZER=fastokens DYN_TOKENIZER_CACHE=1 DYN_TOKENIZER_CACHE_BYTES=8000000000\nexport DYN_TCP_CHANNEL_BUFFER=128 DYN_TCP_REQUEST_TIMEOUT=60\nuv run --no-project python -m dynamo.frontend --http-port 8000 --router-mode kv \\\n --router-session-affinity-ttl-secs 14400 --active-decode-blocks-threshold None \\\n --active-prefill-tokens-threshold None --active-prefill-tokens-threshold-frac None\n```\n\n## Dynamo sidecar serving\n\nDynamo can route requests to vLLM's native gRPC server through a colocated\nsidecar. Choose the checkpoint, hardware and parallelism in the command\nbuilder above. For compatible runtime installation and complete launch\nexamples, see the [Dynamo vLLM sidecar guide](https://github.com/ai-dynamo/dynamo/blob/main/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/vllm/sidecar.md).\n\nThe B300 deployment in [InferenceX #4231](https://github.com/SemiAnalysisAI/InferenceX/pull/4231)\nvalidated this integration with the original\n[DeepSeek-V4.1-Flash checkpoint](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash).\nThe builder defaults to DeepSeek-V4.1-Flash-1017; that checkpoint was not\ncovered by this deployment's validation.\n\n### Connect the engine and sidecar\n\nUse the generated model and hardware arguments with `vllm-rs serve`.\nAdd `--grpc-port 19000` and set `--reasoning-parser None`; omit the engine's\n`--tool-call-parser` and `--enable-auto-tool-choice` flags because the\nsidecar handles reasoning and tool parsing. For KV-aware routing, enable\nKV events on the engine with `--kv-events-config`:\n\n```json\n{\"publisher\":\"zmq\",\"topic\":\"kv-events\",\"enable_kv_cache_events\":true,\"endpoint\":\"tcp://*:5200\"}\n```\n\nWith etcd running, set its reachable address in each frontend and sidecar\nshell, along with the shared request and event planes:\n\n```bash\nexport ETCD_ENDPOINTS=http://ETCD_HOST:2379\nexport DYN_REQUEST_PLANE=tcp DYN_EVENT_PLANE=zmq\n```\n\nIn a separate shell on the engine host, connect one sidecar to the engine:\n\n```bash\nexport DYN_VLLM_KV_EVENT_PORT=5200\nuv run --no-project python -m dynamo.vllm.sidecar \\\n --grpc-endpoint 127.0.0.1:19000 \\\n --dyn-tool-call-parser deepseek_v41 --dyn-reasoning-parser deepseek_v41\n```\n\nStart a shared frontend:\n\n```bash\nuv run --no-project python -m dynamo.frontend --http-port 8000 --router-mode kv\n```\n\nFor disaggregated serving, configure the engines as in\n**Dynamo disaggregated serving on B300 and GB300**. Add `--disaggregation-mode prefill --component prefill` to prefill sidecars\nand `--disaggregation-mode decode` to decode sidecars. Allocate distinct\ngRPC and KV-event ports when running several engines on the same host.\n\nCheck the frontend after the workers register:\n\n```bash\ncurl --fail http://FRONTEND_HOST:8000/health\ncurl --fail http://FRONTEND_HOST:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\"model\":\"deepseek-ai/DeepSeek-V4.1-Flash\",\"messages\":[{\"role\":\"user\",\"content\":\"Hello\"}],\"max_tokens\":32}'\n```\n\nSet the request's `model` to the served model name selected for the engine.\n\n## DGX Station (GB300) Tensor Parallel\n\nOne GB300 exposes 252 GB of HBM next to 496 GB of Grace LPDDR5X, and the\n476 GiB checkpoint does not fit the HBM. The Station row serves TP1 with\nboth Engram tables (189 GiB) and 60 GiB of routed experts in pinned host\nmemory, read through UVA. At `--gpu-memory-utilization 0.97` the KV pool\nholds about 3.09M tokens at full 1M context. 60 GiB is the lowest\nverified offload at 1M context with speculative decoding on (55 fails the\nminimum-KV check). For more concurrent 1M contexts, raise\n`--cpu-offload-gb` to 70 and accept ~10% lower single-stream decode.\n\nMeasured on `vllm/vllm-openai:nightly-dee37d89`:\n\n| | 1 stream | 2 | 4 | 8 | 12 | 16 |\n|---|--:|--:|--:|--:|--:|--:|\n| aggregate tok/s, prose | 90.5 | 124 | 168 | 302 | 378 | 416–423 |\n\n- **Autotune.** The first boot on a new flag set or image runs a 75–90\n minute FlashInfer autotune. Mount `/root/.cache/vllm` on a persistent\n volume; later boots start in about 7 minutes.\n- **Host memory.** With the server loaded, ~430 of the 494 GiB Grace pool\n is in use (pinned weights plus loader shared memory).\n `VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1` frees ~66 GB at about 7%\n lower single-stream decode.\n\nLaunch scripts and full measurements:\n[J-M-Recipes/recipes](https://github.com/J-M-Recipes/recipes/tree/main/recipes/dgx-station-gb300/deepseek-v4.1-flash-vllm-uva-dspark).\n\n## KV cache offloading\n\nOffloading requires vLLM 0.31.0+, the first release containing [vllm-project/vllm#57145](https://github.com/vllm-project/vllm/pull/57145).\nValidation used [deepseek-ai/DeepSeek-V4.1-Flash](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash) on 4×H200 (TP4),\nvLLM commit `d1b4028d7e`, the Python frontend, `--enforce-eager`,\n`--max-model-len 16384`, `--max-num-seqs 16` and `--max-num-batched-tokens 8192`.\nDeepSeek validation used `--gpu-memory-utilization 0.95`.\nValidation used `TieringOffloadingSpec`: 8 GiB total CPU tier, `blocks_per_chunk: 1`,\na persistent filesystem `root_dir`, eight read/write threads and `PYTHONHASHSEED=0`.\nThree CPU-tier reloads and three filesystem reloads after restarting the server passed.\nReloaded responses matched the cold output, including the reasoning field.\nThe five-shot GSM8K diagnostic scored 31/32; this is a small sample, not the full benchmark.\nThese results cover this text-only configuration, not the builder's larger tier/chunk defaults or other variants.\n\n## References\n\n- [Model card](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash)\n- [DeepSeek-V4.1-Flash-1017 model card](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash-1017)\n- [Technical report](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash/blob/main/DeepSeek_V41_Tech_Report.pdf)\n- [DeepSeek-V4-Flash recipe](/deepseek-ai/DeepSeek-V4-Flash) — the V4 sibling this diverges from\n- [DeepSeek-V3.2-Exp recipe](/deepseek-ai/DeepSeek-V3.2-Exp) — where the indexer first appeared\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "d90e35cffa21729ac55bb0477a0d64f5127dbcda", "path": "recipes/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml", "overlays": [] } }