{
  "preset": "mi355x_agentx_dpa",
  "label": "AgentX DPA8 + TP experts",
  "hardware": "mi355x",
  "strategy": "single_node_dpa_tp",
  "variant": "default",
  "workload": "Agentic coding (InferenceX AgentX)",
  "default": false,
  "source": {
    "label": "InferenceX #4526",
    "href": "https://github.com/SemiAnalysisAI/InferenceX/pull/4526",
    "key": "dsv4-fp4-mi355x-vllm-agentic-mtp",
    "revision": "5ec3c995988e5d3632c1ff53e8ae39da3e799892"
  },
  "image": {
    "vllm": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d",
    "model_revision": "72e1d3230f6c080a530b0a1d46f8eb4602340597",
    "pip": [
      "Pillow",
      "fastapi",
      "uvicorn",
      "vllm-router==0.1.14"
    ]
  },
  "default_point": "c64",
  "points": {
    "c64": {
      "label": "64",
      "concurrency": 64,
      "hardware": "mi355x",
      "strategy": "single_node_dpa_tp",
      "variant": "default",
      "node_count": 1,
      "deploy_type": "single_node",
      "preset": "mi355x_agentx_dpa",
      "preset_point": "c64",
      "env": {
        "VLLM_ROCM_USE_AITER": "1",
        "VLLM_ENGINE_READY_TIMEOUT_S": "10800",
        "VLLM_USE_V2_MODEL_RUNNER": "1",
        "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4",
        "VLLM_ROCM_USE_AITER_MOE": "1",
        "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
        "OMP_NUM_THREADS": "1"
      },
      "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d",
      "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n  --trust-remote-code \\\n  --kv-cache-dtype fp8 \\\n  --block-size 256 \\\n  --port 8100 \\\n  --data-parallel-size 8 \\\n  --tensor-parallel-size 1 \\\n  --distributed-executor-backend mp \\\n  --max-num-batched-tokens 8192 \\\n  --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n  --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --async-scheduling \\\n  --gpu-memory-utilization 0.86 \\\n  --moe-backend aiter \\\n  --enable-prefix-caching \\\n  --no-disable-hybrid-kv-cache-manager \\\n  --prefill-schedule-interval 8 \\\n  --long-prefill-token-threshold 16384 \\\n  --max-num-seqs 64 \\\n  --tokenizer-mode deepseek_v4 \\\n  --tool-call-parser deepseek_v4 \\\n  --enable-auto-tool-choice \\\n  --reasoning-parser deepseek_v4 \\\n  --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'",
      "argv": [
        "vllm",
        "serve",
        "deepseek-ai/DeepSeek-V4-Pro-0813",
        "--trust-remote-code",
        "--kv-cache-dtype",
        "fp8",
        "--block-size",
        "256",
        "--port",
        "8100",
        "--data-parallel-size",
        "8",
        "--tensor-parallel-size",
        "1",
        "--distributed-executor-backend",
        "mp",
        "--max-num-batched-tokens",
        "8192",
        "--compilation-config",
        "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}",
        "--revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--tokenizer-revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--async-scheduling",
        "--gpu-memory-utilization",
        "0.86",
        "--moe-backend",
        "aiter",
        "--enable-prefix-caching",
        "--no-disable-hybrid-kv-cache-manager",
        "--prefill-schedule-interval",
        "8",
        "--long-prefill-token-threshold",
        "16384",
        "--max-num-seqs",
        "64",
        "--tokenizer-mode",
        "deepseek_v4",
        "--tool-call-parser",
        "deepseek_v4",
        "--enable-auto-tool-choice",
        "--reasoning-parser",
        "deepseek_v4",
        "--speculative-config",
        "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}"
      ],
      "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n  --security-opt seccomp=unconfined --group-add video \\\n  --privileged --ipc=host -p 8100:8100 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  -e VLLM_ROCM_USE_AITER=1 \\\n  -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n  -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n  -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n  -e VLLM_ROCM_USE_AITER_MOE=1 \\\n  -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n  -e OMP_NUM_THREADS=1 \\\n  vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n  --trust-remote-code \\\n  --kv-cache-dtype fp8 \\\n  --block-size 256 \\\n  --port 8100 \\\n  --data-parallel-size 8 \\\n  --tensor-parallel-size 1 \\\n  --distributed-executor-backend mp \\\n  --max-num-batched-tokens 8192 \\\n  --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n  --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --async-scheduling \\\n  --gpu-memory-utilization 0.86 \\\n  --moe-backend aiter \\\n  --enable-prefix-caching \\\n  --no-disable-hybrid-kv-cache-manager \\\n  --prefill-schedule-interval 8 \\\n  --long-prefill-token-threshold 16384 \\\n  --max-num-seqs 64 \\\n  --tokenizer-mode deepseek_v4 \\\n  --tool-call-parser deepseek_v4 \\\n  --enable-auto-tool-choice \\\n  --reasoning-parser deepseek_v4 \\\n  --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'",
      "docker_argv": [
        "docker",
        "run",
        "--device=/dev/kfd",
        "--device=/dev/dri",
        "--security-opt",
        "seccomp=unconfined",
        "--group-add",
        "video",
        "--privileged",
        "--ipc=host",
        "-p",
        "8100:8100",
        "-v",
        "~/.cache/huggingface:/root/.cache/huggingface",
        "-e",
        "VLLM_ROCM_USE_AITER=1",
        "-e",
        "VLLM_ENGINE_READY_TIMEOUT_S=10800",
        "-e",
        "VLLM_USE_V2_MODEL_RUNNER=1",
        "-e",
        "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4",
        "-e",
        "VLLM_ROCM_USE_AITER_MOE=1",
        "-e",
        "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0",
        "-e",
        "OMP_NUM_THREADS=1",
        "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d",
        "deepseek-ai/DeepSeek-V4-Pro-0813",
        "--trust-remote-code",
        "--kv-cache-dtype",
        "fp8",
        "--block-size",
        "256",
        "--port",
        "8100",
        "--data-parallel-size",
        "8",
        "--tensor-parallel-size",
        "1",
        "--distributed-executor-backend",
        "mp",
        "--max-num-batched-tokens",
        "8192",
        "--compilation-config",
        "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}",
        "--revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--tokenizer-revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--async-scheduling",
        "--gpu-memory-utilization",
        "0.86",
        "--moe-backend",
        "aiter",
        "--enable-prefix-caching",
        "--no-disable-hybrid-kv-cache-manager",
        "--prefill-schedule-interval",
        "8",
        "--long-prefill-token-threshold",
        "16384",
        "--max-num-seqs",
        "64",
        "--tokenizer-mode",
        "deepseek_v4",
        "--tool-call-parser",
        "deepseek_v4",
        "--enable-auto-tool-choice",
        "--reasoning-parser",
        "deepseek_v4",
        "--speculative-config",
        "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}"
      ],
      "companions": [
        {
          "feature": "strategy:single_node_dpa_tp",
          "label": "vLLM Router",
          "after": true,
          "description": "Routes client requests across the DP ranks of the single engine process. --intra-node-data-parallel-size must match the engine's --data-parallel-size; consistent_hash keeps a conversation pinned to the rank that already holds its prefix. Requires: uv pip install vllm-router",
          "command": "vllm-router \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --policy consistent_hash \\\n  --intra-node-data-parallel-size 8 \\\n  --worker-urls http://localhost:8100 \\\n  --worker-startup-timeout-secs 1800"
        }
      ]
    },
    "c128": {
      "label": "128",
      "concurrency": 128,
      "hardware": "mi355x",
      "strategy": "single_node_dpa_tp",
      "variant": "default",
      "node_count": 1,
      "deploy_type": "single_node",
      "preset": "mi355x_agentx_dpa",
      "preset_point": "c128",
      "env": {
        "VLLM_ROCM_USE_AITER": "1",
        "VLLM_ENGINE_READY_TIMEOUT_S": "10800",
        "VLLM_USE_V2_MODEL_RUNNER": "1",
        "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4",
        "VLLM_ROCM_USE_AITER_MOE": "1",
        "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
        "OMP_NUM_THREADS": "1"
      },
      "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d",
      "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n  --trust-remote-code \\\n  --kv-cache-dtype fp8 \\\n  --block-size 256 \\\n  --port 8100 \\\n  --data-parallel-size 8 \\\n  --tensor-parallel-size 1 \\\n  --distributed-executor-backend mp \\\n  --max-num-batched-tokens 8192 \\\n  --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n  --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --async-scheduling \\\n  --gpu-memory-utilization 0.86 \\\n  --moe-backend aiter \\\n  --enable-prefix-caching \\\n  --no-disable-hybrid-kv-cache-manager \\\n  --prefill-schedule-interval 8 \\\n  --long-prefill-token-threshold 16384 \\\n  --max-num-seqs 128 \\\n  --tokenizer-mode deepseek_v4 \\\n  --tool-call-parser deepseek_v4 \\\n  --enable-auto-tool-choice \\\n  --reasoning-parser deepseek_v4 \\\n  --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'",
      "argv": [
        "vllm",
        "serve",
        "deepseek-ai/DeepSeek-V4-Pro-0813",
        "--trust-remote-code",
        "--kv-cache-dtype",
        "fp8",
        "--block-size",
        "256",
        "--port",
        "8100",
        "--data-parallel-size",
        "8",
        "--tensor-parallel-size",
        "1",
        "--distributed-executor-backend",
        "mp",
        "--max-num-batched-tokens",
        "8192",
        "--compilation-config",
        "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}",
        "--revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--tokenizer-revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--async-scheduling",
        "--gpu-memory-utilization",
        "0.86",
        "--moe-backend",
        "aiter",
        "--enable-prefix-caching",
        "--no-disable-hybrid-kv-cache-manager",
        "--prefill-schedule-interval",
        "8",
        "--long-prefill-token-threshold",
        "16384",
        "--max-num-seqs",
        "128",
        "--tokenizer-mode",
        "deepseek_v4",
        "--tool-call-parser",
        "deepseek_v4",
        "--enable-auto-tool-choice",
        "--reasoning-parser",
        "deepseek_v4",
        "--speculative-config",
        "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}"
      ],
      "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n  --security-opt seccomp=unconfined --group-add video \\\n  --privileged --ipc=host -p 8100:8100 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  -e VLLM_ROCM_USE_AITER=1 \\\n  -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n  -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n  -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n  -e VLLM_ROCM_USE_AITER_MOE=1 \\\n  -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n  -e OMP_NUM_THREADS=1 \\\n  vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n  --trust-remote-code \\\n  --kv-cache-dtype fp8 \\\n  --block-size 256 \\\n  --port 8100 \\\n  --data-parallel-size 8 \\\n  --tensor-parallel-size 1 \\\n  --distributed-executor-backend mp \\\n  --max-num-batched-tokens 8192 \\\n  --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n  --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --async-scheduling \\\n  --gpu-memory-utilization 0.86 \\\n  --moe-backend aiter \\\n  --enable-prefix-caching \\\n  --no-disable-hybrid-kv-cache-manager \\\n  --prefill-schedule-interval 8 \\\n  --long-prefill-token-threshold 16384 \\\n  --max-num-seqs 128 \\\n  --tokenizer-mode deepseek_v4 \\\n  --tool-call-parser deepseek_v4 \\\n  --enable-auto-tool-choice \\\n  --reasoning-parser deepseek_v4 \\\n  --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'",
      "docker_argv": [
        "docker",
        "run",
        "--device=/dev/kfd",
        "--device=/dev/dri",
        "--security-opt",
        "seccomp=unconfined",
        "--group-add",
        "video",
        "--privileged",
        "--ipc=host",
        "-p",
        "8100:8100",
        "-v",
        "~/.cache/huggingface:/root/.cache/huggingface",
        "-e",
        "VLLM_ROCM_USE_AITER=1",
        "-e",
        "VLLM_ENGINE_READY_TIMEOUT_S=10800",
        "-e",
        "VLLM_USE_V2_MODEL_RUNNER=1",
        "-e",
        "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4",
        "-e",
        "VLLM_ROCM_USE_AITER_MOE=1",
        "-e",
        "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0",
        "-e",
        "OMP_NUM_THREADS=1",
        "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d",
        "deepseek-ai/DeepSeek-V4-Pro-0813",
        "--trust-remote-code",
        "--kv-cache-dtype",
        "fp8",
        "--block-size",
        "256",
        "--port",
        "8100",
        "--data-parallel-size",
        "8",
        "--tensor-parallel-size",
        "1",
        "--distributed-executor-backend",
        "mp",
        "--max-num-batched-tokens",
        "8192",
        "--compilation-config",
        "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}",
        "--revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--tokenizer-revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--async-scheduling",
        "--gpu-memory-utilization",
        "0.86",
        "--moe-backend",
        "aiter",
        "--enable-prefix-caching",
        "--no-disable-hybrid-kv-cache-manager",
        "--prefill-schedule-interval",
        "8",
        "--long-prefill-token-threshold",
        "16384",
        "--max-num-seqs",
        "128",
        "--tokenizer-mode",
        "deepseek_v4",
        "--tool-call-parser",
        "deepseek_v4",
        "--enable-auto-tool-choice",
        "--reasoning-parser",
        "deepseek_v4",
        "--speculative-config",
        "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}"
      ],
      "companions": [
        {
          "feature": "strategy:single_node_dpa_tp",
          "label": "vLLM Router",
          "after": true,
          "description": "Routes client requests across the DP ranks of the single engine process. --intra-node-data-parallel-size must match the engine's --data-parallel-size; consistent_hash keeps a conversation pinned to the rank that already holds its prefix. Requires: uv pip install vllm-router",
          "command": "vllm-router \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --policy consistent_hash \\\n  --intra-node-data-parallel-size 8 \\\n  --worker-urls http://localhost:8100 \\\n  --worker-startup-timeout-secs 1800"
        }
      ]
    },
    "c192": {
      "label": "192",
      "concurrency": 192,
      "hardware": "mi355x",
      "strategy": "single_node_dpa_tp",
      "variant": "default",
      "node_count": 1,
      "deploy_type": "single_node",
      "preset": "mi355x_agentx_dpa",
      "preset_point": "c192",
      "env": {
        "VLLM_ROCM_USE_AITER": "1",
        "VLLM_ENGINE_READY_TIMEOUT_S": "10800",
        "VLLM_USE_V2_MODEL_RUNNER": "1",
        "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4",
        "VLLM_ROCM_USE_AITER_MOE": "1",
        "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
        "OMP_NUM_THREADS": "1"
      },
      "docker_image": "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d",
      "command": "vllm serve deepseek-ai/DeepSeek-V4-Pro-0813 \\\n  --trust-remote-code \\\n  --kv-cache-dtype fp8 \\\n  --block-size 256 \\\n  --port 8100 \\\n  --data-parallel-size 8 \\\n  --tensor-parallel-size 1 \\\n  --distributed-executor-backend mp \\\n  --max-num-batched-tokens 8192 \\\n  --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n  --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --async-scheduling \\\n  --gpu-memory-utilization 0.86 \\\n  --moe-backend aiter \\\n  --enable-prefix-caching \\\n  --no-disable-hybrid-kv-cache-manager \\\n  --prefill-schedule-interval 8 \\\n  --long-prefill-token-threshold 16384 \\\n  --max-num-seqs 192 \\\n  --tokenizer-mode deepseek_v4 \\\n  --tool-call-parser deepseek_v4 \\\n  --enable-auto-tool-choice \\\n  --reasoning-parser deepseek_v4 \\\n  --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'",
      "argv": [
        "vllm",
        "serve",
        "deepseek-ai/DeepSeek-V4-Pro-0813",
        "--trust-remote-code",
        "--kv-cache-dtype",
        "fp8",
        "--block-size",
        "256",
        "--port",
        "8100",
        "--data-parallel-size",
        "8",
        "--tensor-parallel-size",
        "1",
        "--distributed-executor-backend",
        "mp",
        "--max-num-batched-tokens",
        "8192",
        "--compilation-config",
        "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}",
        "--revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--tokenizer-revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--async-scheduling",
        "--gpu-memory-utilization",
        "0.86",
        "--moe-backend",
        "aiter",
        "--enable-prefix-caching",
        "--no-disable-hybrid-kv-cache-manager",
        "--prefill-schedule-interval",
        "8",
        "--long-prefill-token-threshold",
        "16384",
        "--max-num-seqs",
        "192",
        "--tokenizer-mode",
        "deepseek_v4",
        "--tool-call-parser",
        "deepseek_v4",
        "--enable-auto-tool-choice",
        "--reasoning-parser",
        "deepseek_v4",
        "--speculative-config",
        "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}"
      ],
      "docker_command": "docker run --device=/dev/kfd --device=/dev/dri \\\n  --security-opt seccomp=unconfined --group-add video \\\n  --privileged --ipc=host -p 8100:8100 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  -e VLLM_ROCM_USE_AITER=1 \\\n  -e VLLM_ENGINE_READY_TIMEOUT_S=10800 \\\n  -e VLLM_USE_V2_MODEL_RUNNER=1 \\\n  -e VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 \\\n  -e VLLM_ROCM_USE_AITER_MOE=1 \\\n  -e VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0 \\\n  -e OMP_NUM_THREADS=1 \\\n  vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d deepseek-ai/DeepSeek-V4-Pro-0813 \\\n  --trust-remote-code \\\n  --kv-cache-dtype fp8 \\\n  --block-size 256 \\\n  --port 8100 \\\n  --data-parallel-size 8 \\\n  --tensor-parallel-size 1 \\\n  --distributed-executor-backend mp \\\n  --max-num-batched-tokens 8192 \\\n  --compilation-config '{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n  --revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --tokenizer-revision 72e1d3230f6c080a530b0a1d46f8eb4602340597 \\\n  --async-scheduling \\\n  --gpu-memory-utilization 0.86 \\\n  --moe-backend aiter \\\n  --enable-prefix-caching \\\n  --no-disable-hybrid-kv-cache-manager \\\n  --prefill-schedule-interval 8 \\\n  --long-prefill-token-threshold 16384 \\\n  --max-num-seqs 192 \\\n  --tokenizer-mode deepseek_v4 \\\n  --tool-call-parser deepseek_v4 \\\n  --enable-auto-tool-choice \\\n  --reasoning-parser deepseek_v4 \\\n  --speculative-config '{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}'",
      "docker_argv": [
        "docker",
        "run",
        "--device=/dev/kfd",
        "--device=/dev/dri",
        "--security-opt",
        "seccomp=unconfined",
        "--group-add",
        "video",
        "--privileged",
        "--ipc=host",
        "-p",
        "8100:8100",
        "-v",
        "~/.cache/huggingface:/root/.cache/huggingface",
        "-e",
        "VLLM_ROCM_USE_AITER=1",
        "-e",
        "VLLM_ENGINE_READY_TIMEOUT_S=10800",
        "-e",
        "VLLM_USE_V2_MODEL_RUNNER=1",
        "-e",
        "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4",
        "-e",
        "VLLM_ROCM_USE_AITER_MOE=1",
        "-e",
        "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=0",
        "-e",
        "OMP_NUM_THREADS=1",
        "vllm/vllm-openai-rocm:nightly-e4a5b5537344b103ed10de7764378ef09604ef3d",
        "deepseek-ai/DeepSeek-V4-Pro-0813",
        "--trust-remote-code",
        "--kv-cache-dtype",
        "fp8",
        "--block-size",
        "256",
        "--port",
        "8100",
        "--data-parallel-size",
        "8",
        "--tensor-parallel-size",
        "1",
        "--distributed-executor-backend",
        "mp",
        "--max-num-batched-tokens",
        "8192",
        "--compilation-config",
        "{\"mode\":3,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}",
        "--revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--tokenizer-revision",
        "72e1d3230f6c080a530b0a1d46f8eb4602340597",
        "--async-scheduling",
        "--gpu-memory-utilization",
        "0.86",
        "--moe-backend",
        "aiter",
        "--enable-prefix-caching",
        "--no-disable-hybrid-kv-cache-manager",
        "--prefill-schedule-interval",
        "8",
        "--long-prefill-token-threshold",
        "16384",
        "--max-num-seqs",
        "192",
        "--tokenizer-mode",
        "deepseek_v4",
        "--tool-call-parser",
        "deepseek_v4",
        "--enable-auto-tool-choice",
        "--reasoning-parser",
        "deepseek_v4",
        "--speculative-config",
        "{\"method\":\"dspark\",\"num_speculative_tokens\":6,\"draft_sample_method\":\"probabilistic\"}"
      ],
      "companions": [
        {
          "feature": "strategy:single_node_dpa_tp",
          "label": "vLLM Router",
          "after": true,
          "description": "Routes client requests across the DP ranks of the single engine process. --intra-node-data-parallel-size must match the engine's --data-parallel-size; consistent_hash keeps a conversation pinned to the rank that already holds its prefix. Requires: uv pip install vllm-router",
          "command": "vllm-router \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --policy consistent_hash \\\n  --intra-node-data-parallel-size 8 \\\n  --worker-urls http://localhost:8100 \\\n  --worker-startup-timeout-secs 1800"
        }
      ]
    }
  }
}