{ "preset": "mi355x_agentx_tp", "label": "AgentX TP8 + monokernel", "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "workload": "Agentic coding (InferenceX AgentX)", "default": false, "source": { "label": "InferenceX #4526", "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4526", "key": "dsv4-fp4-mi355x-vllm-agentic-mtp", "revision": "5ec3c995988e5d3632c1ff53e8ae39da3e799892" }, "image": { "vllm": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "model_revision": "72e1d3230f6c080a530b0a1d46f8eb4602340597", "pip": [ "Pillow", "fastapi", "uvicorn" ] }, "default_point": "c1", "points": { "c1": { "label": "1", "concurrency": 1, "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "mi355x_agentx_tp", "preset_point": "c1", "env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "10800", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", "OMP_NUM_THREADS": "1" }, "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 2 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "2", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n -e VLLM_ROCM_USE_AITER_MOE=1 \\\n -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n -e OMP_NUM_THREADS=1 \\\n vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 2 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--device=/dev/kfd", "--device=/dev/dri", "--security-opt", "seccomp=unconfined", "--group-add", "video", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ROCM_USE_AITER=1", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=10800", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4", "-e", "VLLM_ROCM_USE_AITER_MOE=1", "-e", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0", "-e", "OMP_NUM_THREADS=1", "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "2", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ] }, "c4": { "label": "4", "concurrency": 4, "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "mi355x_agentx_tp", "preset_point": "c4", "env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "10800", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", "OMP_NUM_THREADS": "1" }, "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 8 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "8", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n -e VLLM_ROCM_USE_AITER_MOE=1 \\\n -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n -e OMP_NUM_THREADS=1 \\\n vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 8 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--device=/dev/kfd", "--device=/dev/dri", "--security-opt", "seccomp=unconfined", "--group-add", "video", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ROCM_USE_AITER=1", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=10800", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4", "-e", "VLLM_ROCM_USE_AITER_MOE=1", "-e", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0", "-e", "OMP_NUM_THREADS=1", "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "8", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ] }, "c8": { "label": "8", "concurrency": 8, "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "mi355x_agentx_tp", "preset_point": "c8", "env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "10800", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", "OMP_NUM_THREADS": "1" }, "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 16 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "16", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n -e VLLM_ROCM_USE_AITER_MOE=1 \\\n -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n -e OMP_NUM_THREADS=1 \\\n vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 16 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--device=/dev/kfd", "--device=/dev/dri", "--security-opt", "seccomp=unconfined", "--group-add", "video", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ROCM_USE_AITER=1", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=10800", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4", "-e", "VLLM_ROCM_USE_AITER_MOE=1", "-e", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0", "-e", "OMP_NUM_THREADS=1", "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "16", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ] }, "c16": { "label": "16", "concurrency": 16, "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "mi355x_agentx_tp", "preset_point": "c16", "env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "10800", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", "OMP_NUM_THREADS": "1" }, "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 32 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "32", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n -e VLLM_ROCM_USE_AITER_MOE=1 \\\n -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n -e OMP_NUM_THREADS=1 \\\n vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 32 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--device=/dev/kfd", "--device=/dev/dri", "--security-opt", "seccomp=unconfined", "--group-add", "video", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ROCM_USE_AITER=1", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=10800", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4", "-e", "VLLM_ROCM_USE_AITER_MOE=1", "-e", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0", "-e", "OMP_NUM_THREADS=1", "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "32", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ] }, "c32": { "label": "32", "concurrency": 32, "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "mi355x_agentx_tp", "preset_point": "c32", "env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "10800", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", "OMP_NUM_THREADS": "1" }, "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 64 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "64", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n -e VLLM_ROCM_USE_AITER_MOE=1 \\\n -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n -e OMP_NUM_THREADS=1 \\\n vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 64 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--device=/dev/kfd", "--device=/dev/dri", "--security-opt", "seccomp=unconfined", "--group-add", "video", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ROCM_USE_AITER=1", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=10800", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4", "-e", "VLLM_ROCM_USE_AITER_MOE=1", "-e", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0", "-e", "OMP_NUM_THREADS=1", "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "64", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ] }, "c40": { "label": "40", "concurrency": 40, "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "mi355x_agentx_tp", "preset_point": "c40", "env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "10800", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", "OMP_NUM_THREADS": "1" }, "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 80 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "80", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n -e VLLM_ROCM_USE_AITER_MOE=1 \\\n -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n -e OMP_NUM_THREADS=1 \\\n vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 80 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--device=/dev/kfd", "--device=/dev/dri", "--security-opt", "seccomp=unconfined", "--group-add", "video", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ROCM_USE_AITER=1", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=10800", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4", "-e", "VLLM_ROCM_USE_AITER_MOE=1", "-e", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0", "-e", "OMP_NUM_THREADS=1", "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "80", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ] }, "c48": { "label": "48", "concurrency": 48, "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "mi355x_agentx_tp", "preset_point": "c48", "env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "10800", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", "OMP_NUM_THREADS": "1" }, "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 96 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "96", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n -e VLLM_ROCM_USE_AITER_MOE=1 \\\n -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n -e OMP_NUM_THREADS=1 \\\n vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 96 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--device=/dev/kfd", "--device=/dev/dri", "--security-opt", "seccomp=unconfined", "--group-add", "video", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ROCM_USE_AITER=1", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=10800", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4", "-e", "VLLM_ROCM_USE_AITER_MOE=1", "-e", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0", "-e", "OMP_NUM_THREADS=1", "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "96", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ] }, "c56": { "label": "56", "concurrency": 56, "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "mi355x_agentx_tp", "preset_point": "c56", "env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "10800", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", "OMP_NUM_THREADS": "1" }, "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 112 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "112", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n -e VLLM_ROCM_USE_AITER_MOE=1 \\\n -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n -e OMP_NUM_THREADS=1 \\\n vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 112 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--device=/dev/kfd", "--device=/dev/dri", "--security-opt", "seccomp=unconfined", "--group-add", "video", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ROCM_USE_AITER=1", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=10800", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4", "-e", "VLLM_ROCM_USE_AITER_MOE=1", "-e", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0", "-e", "OMP_NUM_THREADS=1", "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "112", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ] }, "c64": { "label": "64", "concurrency": 64, "hardware": "mi355x", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "mi355x_agentx_tp", "preset_point": "c64", "env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ENGINE_READY_TIMEOUT_S": "10800", "VLLM_USE_V2_MODEL_RUNNER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0", "OMP_NUM_THREADS": "1" }, "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 128 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "128", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ], "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n -e VLLM_ROCM_USE_AITER_MOE=1 \\\n -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n -e OMP_NUM_THREADS=1 \\\n vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 256 \\\n --tensor-parallel-size 8 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 8192 \\\n --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n --async-scheduling \\\n --data-parallel-size 1 \\\n --gpu-memory-utilization 0.86 \\\n --moe-backend aiter \\\n --monokernel \\\n --enable-prefix-caching \\\n --no-disable-hybrid-kv-cache-manager \\\n --max-num-seqs 128 \\\n --tokenizer-mode deepseek_v4 \\\n --tool-call-parser deepseek_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser deepseek_v4 \\\n --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'", "docker_argv": [ "docker", "run", "--device=/dev/kfd", "--device=/dev/dri", "--security-opt", "seccomp=unconfined", "--group-add", "video", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ROCM_USE_AITER=1", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=10800", "-e", "VLLM_USE_V2_MODEL_RUNNER=1", "-e", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4", "-e", "VLLM_ROCM_USE_AITER_MOE=1", "-e", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0", "-e", "OMP_NUM_THREADS=1", "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d", "deepseek-ai/DeepSeek-V4-Pro-0813", "--trust-remote-code", "--kv-cache-dtype", "fp8", "--block-size", "256", "--tensor-parallel-size", "8", "--distributed-executor-backend", "mp", "--max-num-batched-tokens", "8192", "--compilation-config", "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--tokenizer-revision", "72e1d3230f6c080a530b0a1d46f8eb4602340597", "--async-scheduling", "--data-parallel-size", "1", "--gpu-memory-utilization", "0.86", "--moe-backend", "aiter", "--monokernel", "--enable-prefix-caching", "--no-disable-hybrid-kv-cache-manager", "--max-num-seqs", "128", "--tokenizer-mode", "deepseek_v4", "--tool-call-parser", "deepseek_v4", "--enable-auto-tool-choice", "--reasoning-parser", "deepseek_v4", "--speculative-config", "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}" ] } } }