{ "hf_id": "deepseek-ai/DeepSeek-V4-Flash", "meta": { "title": "DeepSeek-V4-Flash", "slug": "deepseek-v4-flash", "provider": "DeepSeek", "description": "DeepSeek V4 MoE model with hybrid CSA+HCA attention, manifold-constrained hyper-connections, and three-tier reasoning (Non-think / Think High / Think Max).", "date_added": "2026-07-31", "date_updated": "2026-09-29", "tasks": [ "text" ], "performance_headline": "Compact 284B/13B V4 sibling — single-node 1M-context serving with FP4+FP8 weights, MTP, and DSpark speculative decoding", "related_recipes": [], "hardware": { "h200": "verified", "b200": "verified", "gb200": "verified", "b300": "verified", "gb300": "verified", "dgx_station_gb300": "verified", "dgx_spark_gb10": "verified", "rtx_pro_6000_8x": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified", "ascend_950dt_8x": "verified", "ascend_950pr_8x": "verified", "ascend_910c": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tep", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_USE_RUST_FRONTEND": "1" }, "docker_image": "vllm/vllm-openai:v0.28.0", "command": "vllm serve deepseek-ai/DeepSeek-V4-Flash-0731 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --enable-expert-parallel \\\n --tensor-parallel-size 8 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Flash-0731", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--enable-expert-parallel", "--tensor-parallel-size", "8", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--reasoning-config", "{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n vllm/vllm-openai:v0.28.0 deepseek-ai/DeepSeek-V4-Flash-0731 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --enable-expert-parallel \\\n --tensor-parallel-size 8 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_USE_RUST_FRONTEND=1", "vllm/vllm-openai:v0.28.0", "deepseek-ai/DeepSeek-V4-Flash-0731", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--enable-expert-parallel", "--tensor-parallel-size", "8", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--reasoning-config", "{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}" ], "strategy_spec": { "name": "single_node_tep", "deploy_type": "single_node", "display_name": "Tensor + Expert Parallel", "short_label": "TEP", "orientation": "balanced", "description": "Single-node TEP. TP splits dense layers and EP splits expert layers across local GPUs. TP must be set to GPU count to avoid OOM from replicated dense layers. For MoE models only.\n", "hardware_match": { "min_gpus": 2, "max_gpus": 8, "multi_node": false, "architecture": "moe" }, "vllm_args": [ "--enable-expert-parallel" ], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tp": "/deepseek-ai/DeepSeek-V4-Flash/strategies/single_node_tp.json", "single_node_dep": "/deepseek-ai/DeepSeek-V4-Flash/strategies/single_node_dep.json", "multi_node_dep": "/deepseek-ai/DeepSeek-V4-Flash/strategies/multi_node_dep.json", "pd_cluster": "/deepseek-ai/DeepSeek-V4-Flash/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/deepseek-ai/DeepSeek-V4-Flash/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/deepseek-ai/DeepSeek-V4-Flash/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/deepseek-ai/DeepSeek-V4-Flash/hw/h200.json", "h100": "/deepseek-ai/DeepSeek-V4-Flash/hw/h100.json", "b200": "/deepseek-ai/DeepSeek-V4-Flash/hw/b200.json", "gb200": "/deepseek-ai/DeepSeek-V4-Flash/hw/gb200.json", "b300": "/deepseek-ai/DeepSeek-V4-Flash/hw/b300.json", "gb300": "/deepseek-ai/DeepSeek-V4-Flash/hw/gb300.json", "dgx_station_gb300": "/deepseek-ai/DeepSeek-V4-Flash/hw/dgx_station_gb300.json", "dgx_spark_gb10": "/deepseek-ai/DeepSeek-V4-Flash/hw/dgx_spark_gb10.json", "rtx_pro_6000_8x": "/deepseek-ai/DeepSeek-V4-Flash/hw/rtx_pro_6000_8x.json", "mi300x": "/deepseek-ai/DeepSeek-V4-Flash/hw/mi300x.json", "mi325x": "/deepseek-ai/DeepSeek-V4-Flash/hw/mi325x.json", "mi355x": "/deepseek-ai/DeepSeek-V4-Flash/hw/mi355x.json", "ascend_950dt_8x": "/deepseek-ai/DeepSeek-V4-Flash/hw/ascend_950dt_8x.json", "ascend_950pr_8x": "/deepseek-ai/DeepSeek-V4-Flash/hw/ascend_950pr_8x.json" } }, "model": { "model_id": "deepseek-ai/DeepSeek-V4-Flash", "min_vllm_version": "0.20.0", "default_frontend": "rust", "docker_image": { "amd": "vllm/vllm-openai-rocm:nightly" }, "architecture": "moe", "parameter_count": "284B", "active_parameters": "13B", "context_length": 1048576, "flashinfer_autotune": true, "base_args": [ "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256" ], "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable tool calling with DeepSeek V4 chat template support.", "args": [ "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "Enable reasoning/thinking mode with the DeepSeek V4 reasoning parser.", "args": [ "--reasoning-parser", "deepseek_v4", "--reasoning-config", "{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}" ] }, "spec_decoding": { "description": "Speculative decoding — pick a drafting method.", "default_mode": "mtp", "modes": { "mtp": { "label": "MTP", "description": "Built-in Multi-Token Prediction, 2 draft tokens (1 on Hopper).", "hardware": { "rtx_pro_6000_8x": "unsupported" }, "variants": [ "fp8", "nvfp4", "redhat_nvfp4", "ascend_w8a8" ], "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" ], "hardware_overrides": { "hopper": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" ] }, "npu": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":1,\"enforce_eager\":true}" ] } } }, "dspark": { "label": "DSpark", "description": "DSpark drafting, 7 draft tokens, probabilistic sampling. Requires the DSpark or 0731 checkpoint (Variant row).", "variants": [ "default", "dspark" ], "args": [ "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}" ], "hardware_overrides": { "dgx_spark_gb10": { "args": [ "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"B12X_MLA_SPARSE\"}" ] }, "npu": { "args": [ "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":5,\"enforce_eager\":true}" ] } } }, "dflash": { "label": "DFlash", "description": "DFlash speculative decoding with an externally trained draft model; the served checkpoint is unchanged.", "variants": [ "default" ], "args": [ "--speculative-config", "{\"model\":\"RedHatAI/DeepSeek-V4-Flash-speculator.dflash\",\"num_speculative_tokens\":7,\"method\":\"dflash\"}" ] } } } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "model_id": "deepseek-ai/DeepSeek-V4-Flash-0731", "label": "FP8 (0731)", "precision": "fp8", "vram_minimum_gb": 200, "description": "Official DeepSeek-V4-Flash release (2026-07-31) superseding the preview — substantially stronger agentic performance, with the DSpark draft module attached.", "supported_hardware": [ "arch:sm90", "arch:sm100", "arch:sm103", "arch:gfx942", "mi355x", "rtx_pro_6000_8x", "dgx_spark_gb10", "ascend_950pr_8x", "ascend_950dt_8x" ], "min_vllm_version": "0.25.0", "docker_image": "vllm/vllm-openai:v0.28.0", "default_modes": { "spec_decoding": "dspark" }, "hardware_overrides": { "rtx_pro_6000_8x": { "docker_image": "vllm/vllm-openai:nightly" }, "ascend_950pr_8x": { "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-a5", "install": { "pip": false } }, "ascend_950dt_8x": { "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-a5", "install": { "pip": false } }, "dgx_spark_gb10": { "docker_image": "eugr/spark-vllm-b12x:latest", "install": { "pip": false }, "extra_args": [ "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--gpu-memory-utilization", "0.85", "--enable-prefix-caching", "--default-chat-template-kwargs.thinking=true", "--default-chat-template-kwargs.reasoning_effort=high", "--load-format", "instanttensor", "--moe-backend", "b12x", "--linear-backend", "b12x", "--attention-backend", "B12X_MLA_SPARSE", "--max-cudagraph-capture-size", "64", "--compilation-config", "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}" ], "extra_env": { "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", "CUTE_DSL_ARCH": "sm_121a", "VLLM_USE_AOT_COMPILE": "1", "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_USE_MEGA_AOT_ARTIFACT": "-1", "VLLM_MEMORY_PROFILE_INCLUDE_ATTN": "1", "VLLM_USE_FLASHINFER_SAMPLER": "1", "VLLM_USE_B12X_WO_PROJECTION": "1", "VLLM_USE_B12X_MHC": "1", "VLLM_USE_B12X_FP8_GEMM": "1", "VLLM_USE_B12X_MOE": "1", "VLLM_USE_B12X_SPARSE_INDEXER": "1", "VLLM_USE_V2_MODEL_RUNNER": "1", "B12X_MLA_SM120_UNIFIED": "1", "B12X_MOE_FORCE_A8": "1", "GLOO_SOCKET_IFNAME": "$IFACE_NAME", "MN_IF_NAME": "$IFACE_NAME", "NCCL_IB_DISABLE": "0", "NCCL_IB_HCA": "$IB_IF", "NCCL_IGNORE_CPU_AFFINITY": "1", "NCCL_SOCKET_IFNAME": "$IFACE_NAME", "OMPI_MCA_btl_tcp_if_include": "$IFACE_NAME", "TP_SOCKET_IFNAME": "$IFACE_NAME", "UCX_NET_DEVICES": "$IFACE_NAME" } }, "blackwell": { "extra_args": [ "--moe-backend", "deep_gemm_mega_moe" ] } }, "json": "/deepseek-ai/DeepSeek-V4-Flash-0731.json" }, "fp8": { "label": "FP8 (Preview)", "precision": "fp8", "vram_minimum_gb": 170, "description": "Native FP4+FP8 mixed checkpoint (MoE experts FP4, remaining params FP8)", "hardware_overrides": { "dgx_spark_gb10": { "docker_image": "eugr/spark-vllm-b12x:latest", "install": { "pip": false }, "extra_args": [ "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--gpu-memory-utilization", "0.85", "--enable-prefix-caching", "--default-chat-template-kwargs.thinking=true", "--default-chat-template-kwargs.reasoning_effort=high", "--load-format", "instanttensor", "--moe-backend", "b12x", "--linear-backend", "b12x", "--attention-backend", "B12X_MLA_SPARSE", "--max-cudagraph-capture-size", "64", "--compilation-config", "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}" ], "extra_env": { "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", "CUTE_DSL_ARCH": "sm_121a", "VLLM_USE_AOT_COMPILE": "1", "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_USE_MEGA_AOT_ARTIFACT": "-1", "VLLM_MEMORY_PROFILE_INCLUDE_ATTN": "1", "VLLM_USE_FLASHINFER_SAMPLER": "1", "VLLM_USE_B12X_WO_PROJECTION": "1", "VLLM_USE_B12X_MHC": "1", "VLLM_USE_B12X_FP8_GEMM": "1", "VLLM_USE_B12X_MOE": "1", "VLLM_USE_B12X_SPARSE_INDEXER": "1", "VLLM_USE_V2_MODEL_RUNNER": "1", "B12X_MLA_SM120_UNIFIED": "1", "B12X_MOE_FORCE_A8": "1", "GLOO_SOCKET_IFNAME": "$IFACE_NAME", "MN_IF_NAME": "$IFACE_NAME", "NCCL_IB_DISABLE": "0", "NCCL_IB_HCA": "$IB_IF", "NCCL_IGNORE_CPU_AFFINITY": "1", "NCCL_SOCKET_IFNAME": "$IFACE_NAME", "OMPI_MCA_btl_tcp_if_include": "$IFACE_NAME", "TP_SOCKET_IFNAME": "$IFACE_NAME", "UCX_NET_DEVICES": "$IFACE_NAME" } }, "blackwell": { "extra_args": [ "--moe-backend", "deep_gemm_mega_moe" ] }, "ascend_950dt_8x": { "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-a5", "install": { "pip": false } }, "ascend_950pr_8x": { "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-a5", "install": { "pip": false } } } }, "nvfp4": { "model_id": "nvidia/DeepSeek-V4-Flash-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 170, "description": "NVIDIA modelopt NVFP4 checkpoint (MoE experts NVFP4; attention, shared experts, router head, and MTP stay FP8) for Blackwell GPUs.", "supported_hardware": [ "arch:sm100", "arch:sm103", "rtx_pro_6000_8x", "dgx_spark_gb10" ], "hardware_overrides": { "dgx_spark_gb10": { "docker_image": "eugr/spark-vllm-b12x:latest", "install": { "pip": false }, "extra_args": [ "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--gpu-memory-utilization", "0.85", "--enable-prefix-caching", "--default-chat-template-kwargs.thinking=true", "--default-chat-template-kwargs.reasoning_effort=high", "--load-format", "instanttensor", "--moe-backend", "b12x", "--linear-backend", "b12x", "--attention-backend", "B12X_MLA_SPARSE", "--max-cudagraph-capture-size", "64", "--compilation-config", "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}" ], "extra_env": { "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", "CUTE_DSL_ARCH": "sm_121a", "VLLM_USE_AOT_COMPILE": "1", "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_USE_MEGA_AOT_ARTIFACT": "-1", "VLLM_MEMORY_PROFILE_INCLUDE_ATTN": "1", "VLLM_USE_FLASHINFER_SAMPLER": "1", "VLLM_USE_B12X_WO_PROJECTION": "1", "VLLM_USE_B12X_MHC": "1", "VLLM_USE_B12X_FP8_GEMM": "1", "VLLM_USE_B12X_MOE": "1", "VLLM_USE_B12X_SPARSE_INDEXER": "1", "VLLM_USE_V2_MODEL_RUNNER": "1", "B12X_MLA_SM120_UNIFIED": "1", "B12X_MOE_FORCE_A8": "1", "GLOO_SOCKET_IFNAME": "$IFACE_NAME", "MN_IF_NAME": "$IFACE_NAME", "NCCL_IB_DISABLE": "0", "NCCL_IB_HCA": "$IB_IF", "NCCL_IGNORE_CPU_AFFINITY": "1", "NCCL_SOCKET_IFNAME": "$IFACE_NAME", "OMPI_MCA_btl_tcp_if_include": "$IFACE_NAME", "TP_SOCKET_IFNAME": "$IFACE_NAME", "UCX_NET_DEVICES": "$IFACE_NAME" } } }, "json": "/nvidia/DeepSeek-V4-Flash-NVFP4.json" }, "redhat_nvfp4": { "model_id": "RedHatAI/DeepSeek-V4-Flash-NVFP4-FP8", "precision": "nvfp4", "label": "NVFP4 (RedHat)", "vram_minimum_gb": 170, "description": "RedHatAI NVFP4+FP8 mixed-precision checkpoint for Blackwell GPUs", "json": "/RedHatAI/DeepSeek-V4-Flash-NVFP4-FP8.json" }, "dspark": { "model_id": "deepseek-ai/DeepSeek-V4-Flash-DSpark", "label": "DSpark", "precision": "fp8", "vram_minimum_gb": 200, "description": "Preview FP4+FP8 weights with the DSpark draft module baked in — same base checkpoint as the default variant plus a fused speculative decoder.", "supported_hardware": [ "arch:sm90", "arch:sm100", "arch:sm103", "arch:gfx942", "mi355x", "rtx_pro_6000_8x", "dgx_spark_gb10" ], "min_vllm_version": "0.25.0", "default_modes": { "spec_decoding": "dspark" }, "hardware_overrides": { "dgx_spark_gb10": { "docker_image": "eugr/spark-vllm-b12x:latest", "install": { "pip": false }, "extra_args": [ "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--gpu-memory-utilization", "0.85", "--enable-prefix-caching", "--default-chat-template-kwargs.thinking=true", "--default-chat-template-kwargs.reasoning_effort=high", "--load-format", "instanttensor", "--moe-backend", "b12x", "--linear-backend", "b12x", "--attention-backend", "B12X_MLA_SPARSE", "--max-cudagraph-capture-size", "64", "--compilation-config", "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}" ], "extra_env": { "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", "CUTE_DSL_ARCH": "sm_121a", "VLLM_USE_AOT_COMPILE": "1", "VLLM_USE_BREAKABLE_CUDAGRAPH": "0", "VLLM_USE_MEGA_AOT_ARTIFACT": "-1", "VLLM_MEMORY_PROFILE_INCLUDE_ATTN": "1", "VLLM_USE_FLASHINFER_SAMPLER": "1", "VLLM_USE_B12X_WO_PROJECTION": "1", "VLLM_USE_B12X_MHC": "1", "VLLM_USE_B12X_FP8_GEMM": "1", "VLLM_USE_B12X_MOE": "1", "VLLM_USE_B12X_SPARSE_INDEXER": "1", "VLLM_USE_V2_MODEL_RUNNER": "1", "B12X_MLA_SM120_UNIFIED": "1", "B12X_MOE_FORCE_A8": "1", "GLOO_SOCKET_IFNAME": "$IFACE_NAME", "MN_IF_NAME": "$IFACE_NAME", "NCCL_IB_DISABLE": "0", "NCCL_IB_HCA": "$IB_IF", "NCCL_IGNORE_CPU_AFFINITY": "1", "NCCL_SOCKET_IFNAME": "$IFACE_NAME", "OMPI_MCA_btl_tcp_if_include": "$IFACE_NAME", "TP_SOCKET_IFNAME": "$IFACE_NAME", "UCX_NET_DEVICES": "$IFACE_NAME" } }, "blackwell": { "extra_args": [ "--moe-backend", "deep_gemm_mega_moe" ] } }, "json": "/deepseek-ai/DeepSeek-V4-Flash-DSpark.json" }, "ascend_w8a8": { "model_id": "Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp", "label": "W8A8 (Ascend A3)", "precision": "int8", "vram_minimum_gb": 341, "description": "ModelSlim INT8 W8A8 checkpoint for one Atlas 800 A3 node (16× Ascend 910C), using DP2+TP8. Requires --quantization ascend.", "supported_hardware": [ "ascend_910c" ], "tp": { "ascend_910c": 8 }, "min_vllm_version": "0.23.0", "extra_args": [ "--data-parallel-size", "2", "--quantization", "ascend", "--max-num-seqs", "32", "--max-num-batched-tokens", "10240", "--gpu-memory-utilization", "0.90", "--block-size", "128", "--compilation-config", "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}", "--async-scheduling", "--api-server-count", "1", "--safetensors-load-strategy", "prefetch", "--additional-config", "{\"ascend_compilation_config\":{\"enable_npugraph_ex\":true,\"enable_static_kernel\":false},\"enable_cpu_binding\":true,\"multistream_overlap_shared_expert\":true}" ], "extra_env": { "HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_BUFFSIZE": "2560", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "OMP_PROC_BIND": "false", "OMP_NUM_THREADS": "10", "TASK_QUEUE_ENABLE": "1", "LD_PRELOAD": "/usr/lib/aarch64-linux-gnu/libjemalloc.so.2:$LD_PRELOAD", "ASCEND_RT_VISIBLE_DEVICES": "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15", "VLLM_USE_MODELSCOPE": "True" }, "hardware_overrides": { "ascend_910c": { "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-a3", "install": { "pip": false } } }, "json": "/Eco-Tech/DeepSeek-V4-Flash-w8a8-mtp.json" } }, "kv_offload_support": { "offloading_cpu": "verified", "offloading_fs": "verified" }, "strategy_hardware": { "multi_node_tp": { "dgx_spark_gb10": "supported" }, "single_node_tp": { "npu": "unsupported" }, "single_node_tep": { "npu": "unsupported" }, "multi_node_dep": { "npu": "unsupported" }, "pd_cluster": { "rtx_pro_6000_8x": "unsupported", "npu": "unsupported", "ascend_910c": "supported" } }, "hardware_overrides": { "blackwell": { "extra_args": [ "--attention_config.indexer_kv_dtype", "mxfp4" ] }, "amd": { "extra_args": [ "--distributed-executor-backend", "mp", "--gpu-memory-utilization", "0.9", "--max-num-seqs", "512", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1" } }, "npu": { "extra_env": { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "OMP_PROC_BIND": "false", "OMP_NUM_THREADS": "10", "HCCL_BUFFSIZE": "1024", "ASCEND_RT_VISIBLE_DEVICES": "0,1,2,3" } } }, "hardware_notes": { "dgx_spark_gb10": { "title": "DGX Spark setup required", "body": "This hardware uses the community eugr/spark-vllm-b12x:latest docker image and a 2-node TP launch — the B12X backends the command relies on are not in vLLM main, so there is no pip/wheel path on Spark. Before running the generated command, check the DGX Spark cluster guide for IFACE_NAME, IB_IF and HEAD_IP.", "link": { "label": "DGX Spark cluster guide", "href": "#dgx-spark-gb10-cluster" } }, "ascend_950dt_8x": { "title": "vLLM Ascend A5 image required", "body": "Mixed serving on one 8-card 950DT node uses 4 of 8 NPUs (DP4+EP). FP8 (Preview) runs MTP k=1; FP8 (0731) runs DSpark k=5, verified on 950PR and offered here on the same A5 command. Pin quay.io/ascend/vllm-ascend:v0.23.0-a5 — the NVIDIA pip wheel cannot run here. Set HCCL_BUFFSIZE=1024 before launch. DeepSeek-V4 on 950 also needs the indexer quant_mode fix in vllm-project/vllm-ascend#16104 — without it the image the tag currently ships aborts during warmup with QuantLightningIndexerV2 EZ0021.", "link": { "label": "Ascend 950DT 8x mixed", "href": "#huawei-ascend-950dt-8x" } }, "ascend_950pr_8x": { "title": "vLLM Ascend A5 image required", "body": "Mixed serving on one 8-card 950PR node uses 4 of 8 NPUs (DP4+EP). FP8 (Preview) runs MTP k=1, same command as 950DT; FP8 (0731) runs DSpark k=5. Pin quay.io/ascend/vllm-ascend:v0.23.0-a5 and set HCCL_BUFFSIZE=1024 before launch. DeepSeek-V4 on 950 also needs the indexer quant_mode fix in vllm-project/vllm-ascend#16104 — without it the image the tag currently ships aborts during warmup with QuantLightningIndexerV2 EZ0021.", "link": { "label": "Ascend 950PR 8x mixed", "href": "#huawei-ascend-950pr-8x" } }, "ascend_910c": { "title": "vLLM Ascend A3 image required", "body": "Single-node DP2+TP8 on all 16 NPUs using the W8A8 quantized checkpoint from Eco-Tech. TP8 shards the 284B weights to 35.5 GB per NPU (within the 64 GB limit). Pin quay.io/ascend/vllm-ascend:v0.23.0-a3 — the NVIDIA pip wheel cannot run here. Set HCCL_BUFFSIZE=2560 before launch. The A3 path uses ModelScope to download the W8A8 weights (pip install modelscope). Multi-node PD separation is also available; see the vllm-ascend tutorial.", "link": { "label": "Ascend 910C A3 reference", "href": "https://docs.vllm.ai/projects/ascend/en/latest/tutorials/models/DeepSeek-V4-Flash.html" } } }, "guide": "## Overview\n\nDeepSeek-V4-Flash is a 284B-total / 13B-active MoE model in the V4 preview family.\nIt pairs a **hybrid attention stack** — Compressed Sparse Attention (CSA) + Heavily\nCompressed Attention (HCA) — with **Manifold-Constrained Hyper-Connections (mHC)** to\nreach 27% of V3.2's per-token inference FLOPs and 10% of V3.2's KV cache at 1M\ncontext. Pre-trained on 32T+ tokens; post-training is a two-stage pipeline (domain-\nspecific expert cultivation + unified consolidation via on-policy distillation).\n\nCheckpoint is **FP4+FP8 mixed**: MoE expert weights are stored in FP4 while the\nremaining (attention / norm / router) params stay in FP8.\n\nAn **NVFP4 variant** (`nvidia/DeepSeek-V4-Flash-NVFP4`) is also available — NVIDIA\nmodelopt re-quantizes the MoE experts to standard NVFP4 while attention, shared\nexperts, router head, and MTP stay FP8. Pick it from the **Variant** row; it runs on\nBlackwell GPUs with the FP4 indexer cache. Unlike the native FP8 checkpoint, the\nNVFP4 experts don't support the `deep_gemm_mega_moe` MoE kernel (FP8-only), so it\nruns on the default MoE backend.\n\n## Checkpoints\n\nFour checkpoints are on the **Variant** row. They differ in *which weights you serve*\nand *which draft module ships with them* — the speculative method itself is picked on\nthe **Spec Decoding** row.\n\n| Variant | Repo | Draft | Notes |\n| :-- | :-- | :-- | :-- |\n| **FP8 (0731)** (default) | `deepseek-ai/DeepSeek-V4-Flash-0731` | DSpark | Official release, new weights + DSpark |\n| **FP8 (Preview)** | `deepseek-ai/DeepSeek-V4-Flash` | MTP | Preview FP4+FP8 mixed weights |\n| **NVFP4** | `nvidia/DeepSeek-V4-Flash-NVFP4` | MTP | modelopt re-quant, Blackwell |\n| **DSpark** | `deepseek-ai/DeepSeek-V4-Flash-DSpark` | DSpark | Preview weights + fused DSpark module |\n\n**0731 is the official DeepSeek-V4-Flash release** and the default here, superseding\nthe preview with substantially stronger agentic capability — 82.7 on Terminal Bench\n2.1 and 54.4 on DeepSWE, versus 61.8 and 7.3 for the preview. It beats\nDeepSeek-V4-Pro (Preview) on every agentic benchmark DeepSeek published despite its\nfar smaller activated parameter count. The preview weights remain available as the\n**FP8 (Preview)** variant for reproducing earlier results.\n\n**DSpark** is *not* new weights — it is the preview checkpoint with a speculative\ndecoding module attached (see [DeepSpec](https://github.com/deepseek-ai/DeepSpec)).\nBoth fused checkpoints add a `dspark_*` block to `config.json`; the preview and NVFP4\ncheckpoints have no such block, which is why the DSpark method is offered only on the\ntwo variants that carry the draft. Selecting either one auto-enables **Spec Decoding**\nwith `method: dspark`, emitting:\n\n```\n--speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}'\n```\n\nThe fused checkpoints are ~167 GB on disk versus ~160 GB for the preview — the draft\nmodule is the difference. Both require **vLLM 0.25.0**: DSpark drafting itself landed\nin 0.25.0, but ROCm support for it (MI325X / MI355X) only shipped in 0.26.0. On\nNVIDIA-only deployments 0.25.0 is sufficient, except the 0731 checkpoint\non RTX PRO 6000 (sm_120) with DSpark, which needs `vllm/vllm-openai:nightly`\nand a FlashInfer build containing flashinfer-ai/flashinfer#4380.\n\n## Reasoning modes\n\nThe chat template exposes three reasoning-effort modes:\n\n- **Non-think** — fast, intuitive responses.\n- **Think High** — explicit chain-of-thought for logical analysis and planning.\n- **Think Max** — maximum reasoning effort; requires `--max-model-len >= 393216`\n (384K tokens) to avoid truncation.\n\nRecommended sampling: `temperature = 1.0`, `top_p = 1.0`.\n\nOn the **0731** variant the effort levels are named `low` / `high` / `max`, and\nDeepSeek recommends `top_p = 0.95` for agentic scenarios (`1.0` otherwise) with\n`temperature = 1.0`. Allow up to **384K output tokens** at the `high` and `max`\nlevels.\n\nNote that neither fused checkpoint ships a Jinja chat template — the repos provide an\n`encoding/` folder with `encode_messages` / `parse_message_from_completion_text`\nhelpers instead. Serving through vLLM with `--tokenizer-mode deepseek_v4` (the\n**Tool Calling** pill) applies the built-in DeepSeek-V4 encoding, so the OpenAI-\ncompatible endpoint works without the helper scripts.\n\n### OpenAI Client Example\n\nFor DeepSeek-V4, keep reasoning controls in `chat_template_kwargs`, as it exposes a\ncustom **Think Max** mode via `\"reasoning_effort\": \"max\"`.\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nmodel = \"deepseek-ai/DeepSeek-V4-Flash\"\nmessages = [{\"role\": \"user\", \"content\": \"What is 17*19? Return only the final integer.\"}]\n\n# Non-think\nresp = client.chat.completions.create(\n model=model,\n messages=messages,\n)\n\n# Think High\nresp = client.chat.completions.create(\n model=model,\n messages=messages,\n extra_body={\n \"chat_template_kwargs\": {\n \"thinking\": True,\n \"reasoning_effort\": \"high\",\n },\n },\n)\n\n# Think Max\nresp = client.chat.completions.create(\n model=model,\n messages=messages,\n extra_body={\n \"chat_template_kwargs\": {\n \"thinking\": True,\n \"reasoning_effort\": \"max\",\n },\n },\n)\n```\n\n## Recommended deployment\n\nNon-disaggregated serving on multi-GPU supported hardware: single-node DP + EP with\n`--data-parallel-size 4`. Fills a GB200 NVL4 tray exactly; uses 4 of 8 GPUs per\nreplica on H200/B200/B300 (leaving headroom for throughput-vs-latency tuning).\nOn DGX Station, use the single-GPU launch below. For disaggregated prefill/decode\non GB200, use the PD Cluster tab.\n\n### H100 (8×80 GB)\n\nThe FP8 (0731) and DSpark checkpoints are selectable on H100. A reported\n8×H100 TP8 + EP deployment of 0731 served requests with vLLM 0.26.0, while\nthe same deployment failed during startup on 0.27.0, both with and without\nDSpark. See [vLLM #52065](https://github.com/vllm-project/vllm/issues/52065).\n\nFor the separate DSpark checkpoint, a 4×H100 TP4 report on vLLM 0.25.0\nreached inference after removing `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`\nand setting `--gpu-memory-utilization` below 0.9. A 2×H100 NVL report also\nran the checkpoint with DSpark k=5 on a custom vLLM build. These reports\nestablish Hopper compatibility, but do not verify this recipe's generated\nH100 topology or current image. Check startup and inference before production.\nSee [vLLM #49101](https://github.com/vllm-project/vllm/issues/49101) and\n[the H100 NVL report](https://github.com/vllm-project/vllm/pull/46995#issuecomment-4835994467).\n\n### DGX Spark (GB10) cluster\n\nDGX Spark requires the community Spark vLLM build for this model rather than the\nstock vLLM release/nightly image. The **DGX Spark (GB10)** hardware pill pins the\nDocker tab to `eugr/spark-vllm-b12x:latest`, whose build carries the B12X\nbackends and Spark-specific kernels for the GB10 / SM12x family.\n\nUse **2 nodes**. A single GB10's 128 GB unified memory is below the FP8/NVFP4\ncheckpoint footprint, so the recipe grants DGX Spark a Spark-only **Multi-Node TP**\nlayout (`--tensor-parallel-size 2`, one GB10 per node). The generated command\nalso sets the B12X runtime environment and the Spark network environment; export\n`IFACE_NAME`, `IB_IF` and `HEAD_IP` on each node before launching.\n\nTo identify the values, first list the ConnectX Ethernet interfaces, RoCE\ndevices and the IP address of the UP ConnectX interface on each Spark:\n\n```bash\n$ ibdev2netdev\nroceP2p1s0f0 port 1 ==> enP2p1s0f0np0 (Up)\nroceP2p1s0f1 port 1 ==> enP2p1s0f1np1 (Down)\nrocep1s0f0 port 1 ==> enp1s0f0np0 (Up)\nrocep1s0f1 port 1 ==> enp1s0f1np1 (Down)\n\n$ ip addr show enp1s0f0np0\n3: enp1s0f0np0: mtu 1500 qdisc mq state UP group default qlen 1000\n link/ether 4c:62:56:c1:83:68 brd ff:ff:ff:ff:ff:ff\n inet 100.100.100.10/24 brd 100.100.100.255 scope global noprefixroute enp1s0f0np0\n valid_lft forever preferred_lft forever\n```\n\nFrom above example values, we can get the following common values:\n\n```bash\n# Common values for all nodes\nHEAD_IP=\"100.100.100.10\" # IP of the head node's UP ConnectX interface\nIFACE_NAME=\"enp1s0f0np0\" # The head node's UP ConnectX Ethernet interface\nIB_IF=\"rocep1s0f0,roceP2p1s0f0\" # Comma-separated RoCE device name list corresponding to the UP ConnectX interface\n```\n\n### DGX Station Single-GPU\n\n```bash\nvllm serve deepseek-ai/DeepSeek-V4-Flash \\\n --tensor-parallel-size 1 --pipeline-parallel-size 1 \\\n --kv-cache-dtype fp8 --trust-remote-code --block-size 256 \\\n --gpu-memory-utilization 0.92 \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --attention_config.indexer_kv_dtype mxfp4 \\\n --tokenizer-mode deepseek_v4 --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice --reasoning-parser deepseek_v4 \\\n --max-cudagraph-capture-size 128 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}'\n```\n\n### RTX PRO 6000 8× (8×96 GB, sm_120)\n\nSingle-node TP8 + expert parallelism on the **0731** checkpoint. Serving\nwithout speculative decoding was verified on 8× RTX PRO 6000 (PCIe, no\nNVLink). DSpark on **0731** needs `vllm/vllm-openai:nightly` (or a build\nthat includes vLLM #51538) plus a FlashInfer that instantiates the SM120\nsparse-MLA decode kernel for `topk=192`. The Preview, NVFP4, and DSpark\nvariant checkpoints are not verified on this profile.\n\nsm_120 cannot use the SM100 FP4 indexer cache or `deep_gemm_mega_moe`.\nThe generated command already sets `--attention_config.indexer_kv_dtype\nfp8` and `--moe-backend auto`. Do not use `method: mtp` on 0731 — that\ncheckpoint has no MTP head; Spec Decoding defaults to DSpark.\n\nHost-side FlashInfer install before launching 0731 with DSpark (`--no-deps`\nis required so pip does not replace vLLM's torch):\n\n```bash\n# Trace: flashinfer-ai/flashinfer#4380\n# merge: 24d7dfb2639083c5a4d418881099421fc800b7bb (2026-08-08)\n# wheel: flashinfer-python==0.6.18.dev20260814\n# index: https://flashinfer.ai/whl/nightly/\n/usr/bin/python3.12 -m pip install -U --pre flashinfer-python \\\n--index-url https://flashinfer.ai/whl/nightly/ --no-deps\n/usr/bin/python3.12 -m pip install flashinfer-python\n/usr/bin/python3.12 -m pip install -U --pre flashinfer-cubin \\\n--index-url https://flashinfer.ai/whl/nightly/\n/usr/bin/python3.12 -m pip install -U --pre flashinfer-jit-cache \\\n--index-url https://flashinfer.ai/whl/nightly/cu130\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash-0731 \\\n--trust-remote-code \\\n--kv-cache-dtype fp8 \\\n--block-size 256 \\\n--enable-expert-parallel \\\n--tensor-parallel-size 8 \\\n--moe-backend auto \\\n--attention_config.indexer_kv_dtype fp8 \\\n--tokenizer-mode deepseek_v4 \\\n--tool-call-parser deepseek_v4 \\\n--enable-auto-tool-choice \\\n--reasoning-parser deepseek_v4 \\\n--reasoning-config '{\"reasoning_parser\":\"deepseek_v4\",\"reasoning_start_str\":\"\",\"reasoning_end_str\":\"\"}' \\\n--speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\"}'\n```\n\n### MI300X (8×192GB)\n\nVerified single-node serving on MI300X (8×192 GB, CDNA 3) with the same AMD\nlaunch as the MI325X/MI355X sections below — the builder lowers\n`--gpu-memory-utilization` to `0.85` on MI300X (the 192 GB cards leave less\nheadroom at `0.9`), and DSpark speculative decoding is validated on it.\n\n### MI325X (1×256GB)\n\nValidated end-to-end with vLLM 0.25.0 and PyTorch HIP 7.2.53211. Use this\nconservative TP1 eager-mode launch instead of the MI355X throughput settings:\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash \\\n --host 0.0.0.0 \\\n --port 8002 \\\n --tensor-parallel-size 1 \\\n --kv-cache-dtype fp8_e4m3 \\\n --max-model-len 4096 \\\n --enable-chunked-prefill \\\n --max-num-batched-tokens 256 \\\n --kv-cache-memory-bytes 10000000000 \\\n --distributed-executor-backend mp \\\n --trust-remote-code \\\n --tokenizer-mode deepseek_v4 \\\n --moe-backend triton_unfused \\\n --enforce-eager\n```\n\nChunked prefill allows 4K requests while scheduling at most 256 tokens per\nbatch. This configuration loaded the 148.66 GiB checkpoint and passed model-list\nand chat probes in non-thinking and Think High modes during a 24-hour allocation.\nOnly TP1 at 4K context is verified; TP2+ did not validate successfully. Think Max\nrequires at least 384K context and is not available with this configuration.\n\n### MI355X (4×288GB)\n\nIf you want to further lower the latency, you can run with `--tensor-parallel-size 8`.\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\n\nvllm serve deepseek-ai/DeepSeek-V4-Flash \\\n --host localhost \\\n --port 8001 \\\n --dtype auto \\\n --kv-cache-dtype fp8 \\\n --tensor-parallel-size 4 \\\n --max-num-seqs 512 \\\n --max-num-batched-tokens 8192 \\\n --distributed-executor-backend mp \\\n --trust-remote-code \\\n --gpu-memory-utilization 0.9 \\\n --tokenizer-mode deepseek_v4 \\\n --reasoning-parser deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --compilation-config '{\"mode\": 3, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"}'\n```\n\n#### MI355X is validated on GSM8K dataset:\n\n##### Launch command\n\n```bash\nMODEL=deepseek-ai/DeepSeek-V4-Flash\nlm_eval --model local-completions \\\n --model_args model=$MODEL,base_url=http://0.0.0.0:8001/v1/completions,num_concurrent=128,max_retries=10,max_gen_toks=2048,timeout=60000 \\\n --batch_size auto \\\n --tasks gsm8k \\\n --num_fewshot 8 \\\n --output_path . 2>&1 | tee -a eval.log\n```\n\n##### Reported result\n\n```text\nlocal-completions ({'model': 'deepseek-ai/DeepSeek-V4-Flash', 'base_url': 'http://0.0.0.0:8001/v1/completions', 'num_concurrent': 128, 'max_retries': 10, 'max_gen_toks': 2048, 'timeout': 60000}), gen_kwargs: ({}), limit: None, num_fewshot: 8, batch_size: auto\n|Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr|\n|-----|------:|----------------|-----:|-----------|---|-----:|---|-----:|\n|gsm8k| 3|flexible-extract| 8|exact_match|↑ |0.9439|± |0.0063|\n| | |strict-match | 8|exact_match|↑ |0.9431|± |0.0064|\n```\n\n### H200 Single-Node PD (Mooncake)\n\nSingle-host disaggregated serving: 4 prefill GPUs + 4 decode GPUs on one 8-GPU H200 node,\nusing MooncakeConnector over RDMA for KV cache transfer.\n\n**Prefill** (GPUs 0–3, port 8000):\n\n```bash\ndocker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n --network host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v /mnt/shared:/mnt/shared \\\n -e TILELANG_CLEANUP_TEMP_FILES=1 \\\n -e VLLM_DISABLE_COMPILE_CACHE=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \\\n -e VLLM_LOG_STATS_INTERVAL=1 \\\n -e VLLM_MOONCAKE_BOOTSTRAP_PORT=8998 \\\n -e CUDA_VISIBLE_DEVICES=0,1,2,3 \\\n vllm/vllm-openai:v0.28.0 \\\n deepseek-ai/DeepSeek-V4-Flash \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --port 8000 \\\n --data-parallel-size 4 \\\n --enable-expert-parallel \\\n --tokenizer-mode deepseek_v4 \\\n --reasoning-parser deepseek_v4 \\\n --max-model-len auto \\\n --max-num-batched-tokens 16384 \\\n --max-num-seqs 8 \\\n --no-disable-hybrid-kv-cache-manager \\\n --disable-uvicorn-access-log \\\n --kv-transfer-config '{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"mooncake_protocol\":\"rdma\"}}'\n```\n\n**Decode** (GPUs 4–7, port 8001):\n\n```bash\ndocker run --gpus all \\\n --privileged --ipc=host -p 8001:8001 \\\n --network host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v /mnt/shared:/mnt/shared \\\n -e TILELANG_CLEANUP_TEMP_FILES=1 \\\n -e VLLM_DISABLE_COMPILE_CACHE=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \\\n -e VLLM_LOG_STATS_INTERVAL=1 \\\n -e VLLM_MOONCAKE_BOOTSTRAP_PORT=9889 \\\n -e CUDA_VISIBLE_DEVICES=4,5,6,7 \\\n vllm/vllm-openai:v0.28.0 \\\n deepseek-ai/DeepSeek-V4-Flash \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --port 8001 \\\n --data-parallel-size 4 \\\n --enable-expert-parallel \\\n --tokenizer-mode deepseek_v4 \\\n --reasoning-parser deepseek_v4 \\\n --max-model-len auto \\\n --max-num-seqs 512 \\\n --max-num-batched-tokens 512 \\\n --compilation-config '{\"mode\":0,\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"max_cudagraph_capture_size\":512,\"compile_ranges_endpoints\":[512]}' \\\n --no-disable-hybrid-kv-cache-manager \\\n --disable-uvicorn-access-log \\\n --kv-transfer-config '{\"kv_connector\":\"MooncakeConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\",\"kv_buffer_device\":\"cuda\",\"kv_connector_extra_config\":{\"enforce_handshake_compat\":false,\"mooncake_protocol\":\"rdma\"}}'\n```\n\n**[Router](https://github.com/vllm-project/router)**:\n\n```bash\npip install vllm-router\n\nvllm-router --policy round_robin \\\n --vllm-pd-disaggregation \\\n --prefill http://localhost:8000 \\\n --decode http://localhost:8001 \\\n --host 127.0.0.1 \\\n --port 30000 \\\n --intra-node-data-parallel-size 4 \\\n --kv-connector mooncake\n```\n\n### H200 Single-Node PD (Nixl)\n\nSingle-host disaggregated serving: 4 prefill GPUs + 4 decode GPUs on one 8-GPU H200 node,\nusing NixlConnector for KV cache transfer.\n\n**Prefill** (GPUs 0–3, port 8000):\n\n```bash\ndocker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n --network host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v /mnt/shared:/mnt/shared \\\n -e TILELANG_CLEANUP_TEMP_FILES=1 \\\n -e VLLM_DISABLE_COMPILE_CACHE=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \\\n -e VLLM_LOG_STATS_INTERVAL=1 \\\n -e VLLM_NIXL_SIDE_CHANNEL_PORT=5557 \\\n -e CUDA_VISIBLE_DEVICES=0,1,2,3 \\\n vllm/vllm-openai:v0.28.0 \\\n deepseek-ai/DeepSeek-V4-Flash \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --port 8000 \\\n --data-parallel-size 4 \\\n --enable-expert-parallel \\\n --tokenizer-mode deepseek_v4 \\\n --reasoning-parser deepseek_v4 \\\n --max-model-len auto \\\n --max-num-batched-tokens 16384 \\\n --max-num-seqs 8 \\\n --no-disable-hybrid-kv-cache-manager \\\n --disable-uvicorn-access-log \\\n --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\"}'\n```\n\n**Decode** (GPUs 4–7, port 8001):\n\n```bash\ndocker run --gpus all \\\n --privileged --ipc=host -p 8001:8001 \\\n --network host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -v /mnt/shared:/mnt/shared \\\n -e TILELANG_CLEANUP_TEMP_FILES=1 \\\n -e VLLM_DISABLE_COMPILE_CACHE=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \\\n -e VLLM_LOG_STATS_INTERVAL=1 \\\n -e VLLM_NIXL_SIDE_CHANNEL_PORT=5558 \\\n -e CUDA_VISIBLE_DEVICES=4,5,6,7 \\\n vllm/vllm-openai:v0.28.0 \\\n deepseek-ai/DeepSeek-V4-Flash \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --port 8001 \\\n --data-parallel-size 4 \\\n --enable-expert-parallel \\\n --tokenizer-mode deepseek_v4 \\\n --reasoning-parser deepseek_v4 \\\n --max-model-len auto \\\n --max-num-seqs 512 \\\n --max-num-batched-tokens 512 \\\n --compilation-config '{\"mode\":0,\"cudagraph_mode\":\"FULL_DECODE_ONLY\",\"max_cudagraph_capture_size\":512,\"compile_ranges_endpoints\":[512]}' \\\n --no-disable-hybrid-kv-cache-manager \\\n --disable-uvicorn-access-log \\\n --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\"}'\n```\n\n**[Router](https://github.com/vllm-project/router)**:\n\n```bash\npip install vllm-router\n\nvllm-router --policy round_robin \\\n --vllm-pd-disaggregation \\\n --prefill http://localhost:8000 \\\n --decode http://localhost:8001 \\\n --host 127.0.0.1 \\\n --port 30000 \\\n --intra-node-data-parallel-size 4 \\\n --kv-connector nixl\n```\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "d90e35cffa21729ac55bb0477a0d64f5127dbcda", "path": "recipes/models/deepseek-ai/DeepSeek-V4-Flash.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/deepseek-ai/DeepSeek-V4-Flash.yaml", "overlays": [ { "source": "vllm-ascend", "repo": "vllm-project/vllm-ascend", "ref": "main", "sha": "02ac27a19cb7676304a9ca6985ba56ce29664536", "path": "recipes/overlays/deepseek-ai/DeepSeek-V4-Flash.yaml", "edit_url": "https://github.com/vllm-project/vllm-ascend/edit/main/recipes/overlays/deepseek-ai/DeepSeek-V4-Flash.yaml" } ] } }