{
  "hardware_profiles": {
    "h100": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H100",
      "description": "NVIDIA H100 80GB NVLink (8-GPU node)",
      "gpu_count": 8,
      "vram_gb": 640,
      "multi_node": false
    },
    "h200": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H200",
      "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node",
      "gpu_count": 8,
      "vram_gb": 1128,
      "multi_node": false
    },
    "b200": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm100",
      "display_name": "B200",
      "description": "NVIDIA B200 SXM 180 GB HBM3e · 8-GPU HGX B200 node",
      "gpu_count": 8,
      "vram_gb": 1440,
      "multi_node": false
    },
    "gb200": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm100",
      "display_name": "GB200 NVL4",
      "description": "NVIDIA GB200 Grace-Blackwell compute tray · 192 GB HBM3e/GPU · 4 Blackwell GPUs (NVL72 tray unit)",
      "gpu_count": 4,
      "vram_gb": 768,
      "multi_node": false
    },
    "b300": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm103",
      "display_name": "B300",
      "description": "NVIDIA B300 SXM 268 GB HBM3e · 8-GPU HGX B300 node (Blackwell Ultra)",
      "gpu_count": 8,
      "vram_gb": 2144,
      "multi_node": false
    },
    "gb300": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm103",
      "display_name": "GB300 NVL4",
      "description": "NVIDIA GB300 Grace-Blackwell Ultra compute tray · 288 GB HBM3e/GPU · 4 GPUs (NVL72 tray unit)",
      "gpu_count": 4,
      "vram_gb": 1152,
      "multi_node": false
    },
    "dgx_station_gb300": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm103",
      "display_name": "DGX Station (GB300)",
      "description": "NVIDIA DGX Station · 1× GB300 Grace-Blackwell Ultra Superchip · 252 GB HBM3e (desktop workstation)",
      "gpu_count": 1,
      "vram_gb": 252,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "dgx_spark_gb10": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm121",
      "display_name": "DGX Spark (GB10)",
      "description": "NVIDIA DGX Spark · 1× GB10 Grace Blackwell Superchip · 128 GB unified memory",
      "gpu_count": 1,
      "vram_gb": 128,
      "multi_node": false,
      "restricted": true
    },
    "rtx_pro_6000": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX Pro 6000",
      "description": "NVIDIA RTX Pro 6000 · 1× RTX Pro 6000 GPU · 96 GB VRAM",
      "gpu_count": 1,
      "vram_gb": 96,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_6000_2x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX Pro 6000 2x",
      "description": "NVIDIA RTX Pro 6000 · 2× RTX Pro 6000 GPU · 192 GB VRAM",
      "gpu_count": 2,
      "vram_gb": 192,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_6000_4x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX Pro 6000 4x",
      "description": "NVIDIA RTX Pro 6000 · 4× RTX Pro 6000 GPU · 384 GB VRAM",
      "gpu_count": 4,
      "vram_gb": 384,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_5000": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX PRO 5000",
      "description": "NVIDIA RTX PRO 5000 Blackwell · 72 GB VRAM",
      "gpu_count": 1,
      "vram_gb": 72,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_5000_4x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX PRO 5000 Blackwell 4x",
      "description": "NVIDIA RTX PRO 5000 Blackwell · 4× 72 GB VRAM · PCIe, no NVLink",
      "gpu_count": 4,
      "vram_gb": 288,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_pro_6000_8x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX Pro 6000 8x",
      "description": "NVIDIA RTX Pro 6000 · 8× RTX Pro 6000 GPU · 768 GB VRAM",
      "gpu_count": 8,
      "vram_gb": 768,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_5090": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX 5090",
      "description": "NVIDIA RTX 5090 · 1× RTX 5090 GPU · 32 GB VRAM",
      "gpu_count": 1,
      "vram_gb": 32,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_4090_2x": {
      "brand": "NVIDIA",
      "generation": "ada",
      "compute_arch": "sm89",
      "display_name": "RTX 4090",
      "description": "NVIDIA RTX 4090 · 2× RTX 4090 GPU · 24 GB VRAM/GPU",
      "gpu_count": 2,
      "vram_gb": 48,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "rtx_5090_2x": {
      "brand": "NVIDIA",
      "generation": "blackwell",
      "compute_arch": "sm120",
      "display_name": "RTX 5090",
      "description": "NVIDIA RTX 5090 · 2× RTX 5090 GPU · 32 GB VRAM/GPU",
      "gpu_count": 2,
      "vram_gb": 64,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "l40s": {
      "brand": "NVIDIA",
      "generation": "ada",
      "compute_arch": "sm89",
      "display_name": "L40S",
      "description": "NVIDIA L40S · 1× L40S GPU · 48 GB GDDR6 ECC (PCIe, no NVLink)",
      "gpu_count": 1,
      "vram_gb": 48,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "mi300x": {
      "brand": "AMD",
      "generation": "amd",
      "compute_arch": "gfx942",
      "display_name": "MI300X",
      "description": "AMD Instinct MI300X 192 GB HBM3 · 8-GPU OAM node (CDNA 3)",
      "gpu_count": 8,
      "vram_gb": 1536,
      "multi_node": false
    },
    "mi325x": {
      "brand": "AMD",
      "generation": "amd",
      "compute_arch": "gfx942",
      "display_name": "MI325X",
      "description": "AMD Instinct MI325X 256 GB HBM3e · 8-GPU OAM node (CDNA 3)",
      "gpu_count": 8,
      "vram_gb": 2048,
      "multi_node": false
    },
    "mi350x": {
      "brand": "AMD",
      "generation": "amd",
      "compute_arch": "gfx950",
      "display_name": "MI350X",
      "description": "AMD Instinct MI350X 288 GB HBM3e · 8-GPU OAM node (CDNA 4)",
      "gpu_count": 8,
      "vram_gb": 2304,
      "multi_node": false
    },
    "mi355x": {
      "brand": "AMD",
      "generation": "amd",
      "compute_arch": "gfx950",
      "display_name": "MI355X",
      "description": "AMD Instinct MI355X 288 GB HBM3e · 8-GPU OAM node (CDNA 4)",
      "gpu_count": 8,
      "vram_gb": 2304,
      "multi_node": false
    },
    "trillium": {
      "brand": "Google",
      "generation": "tpu",
      "display_name": "TPU v6e (Trillium)",
      "description": "Google Cloud TPU v6e Trillium · 32 GB HBM/chip · served via vLLM TPU",
      "gpu_count": 8,
      "vram_gb": 256,
      "multi_node": false,
      "restricted": true
    },
    "ironwood": {
      "brand": "Google",
      "generation": "tpu",
      "display_name": "TPU v7 (Ironwood)",
      "description": "Google Cloud TPU v7 Ironwood · 192 GB HBM/chip · served via vLLM TPU",
      "gpu_count": 1,
      "vram_gb": 192,
      "multi_node": false,
      "restricted": true
    },
    "xeon6": {
      "brand": "Intel",
      "generation": "cpu",
      "display_name": "Xeon 6",
      "description": "Intel Xeon 6 CPU NUMA node · x86 CPU backend",
      "gpu_count": 4,
      "vram_gb": 1024,
      "multi_node": false,
      "restricted": true
    },
    "xeon5": {
      "brand": "Intel",
      "generation": "cpu",
      "display_name": "Xeon 5",
      "description": "Intel Xeon 5 CPU NUMA node · x86 CPU backend",
      "gpu_count": 4,
      "vram_gb": 1024,
      "multi_node": false,
      "restricted": true
    },
    "arc_pro_b60": {
      "brand": "Intel",
      "generation": "xpu",
      "display_name": "Arc Pro B60",
      "description": "Intel Arc Pro B60 24 GB · 4-card node · vLLM XPU backend",
      "gpu_count": 4,
      "vram_gb": 96,
      "multi_node": false,
      "restricted": true
    },
    "arc_pro_b70": {
      "brand": "Intel",
      "generation": "xpu",
      "display_name": "Arc Pro B70",
      "description": "Intel Arc Pro B70 32 GB · 4-card node · vLLM XPU backend",
      "gpu_count": 4,
      "vram_gb": 128,
      "multi_node": false,
      "restricted": true
    },
    "ascend_950pr": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 950PR",
      "description": "Huawei Ascend 950PR · 1× NPU · 128 GB HiBL 1.0 HBM · vLLM Ascend backend",
      "gpu_count": 1,
      "vram_gb": 128,
      "multi_node": false,
      "scalable": false,
      "restricted": true
    },
    "ascend_950dt": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 950DT",
      "description": "Huawei Ascend 950DT · 8× NPU node · 96 GB HBM each · vLLM Ascend backend",
      "gpu_count": 8,
      "vram_gb": 768,
      "multi_node": false,
      "restricted": true
    },
    "ascend_910c": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 910C",
      "description": "Huawei Ascend 910C · 16× NPU · 64 GB HBM each · Atlas 800 A3 · vLLM Ascend backend",
      "gpu_count": 16,
      "vram_gb": 1024,
      "multi_node": false,
      "restricted": true
    },
    "ascend_950dt_8x": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 950DT 8x",
      "description": "Huawei Ascend 950DT · 8× NPU · 96 GB HBM each · Atlas A5 · vLLM Ascend backend",
      "gpu_count": 8,
      "vram_gb": 768,
      "multi_node": false,
      "restricted": true
    },
    "ascend_950pr_8x": {
      "brand": "Huawei",
      "generation": "npu",
      "display_name": "Ascend 950PR 8x",
      "description": "Huawei Ascend 950PR · 8× NPU · 128 GB HiBL 1.0 HBM each · Atlas A5 · vLLM Ascend backend",
      "gpu_count": 8,
      "vram_gb": 1024,
      "multi_node": false,
      "restricted": true
    }
  },
  "tasks": [
    {
      "id": "text",
      "display_name": "Text",
      "description": "Text-only models: chat, reasoning, code generation, tool calling"
    },
    {
      "id": "multimodal",
      "display_name": "Multimodal",
      "description": "Vision, audio, or other modalities beyond text"
    },
    {
      "id": "omni",
      "display_name": "Omni",
      "description": "Any-to-any models: text + vision + audio + generation"
    },
    {
      "id": "embedding",
      "display_name": "Embedding",
      "description": "Embedding, reranker, classifier, scoring models"
    }
  ],
  "strategies": [
    "single_node_tp",
    "single_node_tep",
    "single_node_dep",
    "multi_node_tp",
    "multi_node_tep",
    "multi_node_dep",
    "multi_node_tp_pp",
    "multi_node_tp_dp",
    "pd_cluster"
  ],
  "frontends": {
    "rust": {
      "label": "Rust",
      "env": {
        "VLLM_USE_RUST_FRONTEND": "1"
      }
    },
    "python": {
      "label": "Python"
    }
  },
  "kv_offload_groups": {
    "offloading": {
      "display_name": "Offloading Connector",
      "label": "Offloading",
      "order": 2,
      "sub_row_label": "Offload Tiers",
      "docs_url": "https://docs.vllm.ai/en/stable/features/kv_offloading_usage",
      "tooltip": "Offloading Connector — vLLM's built-in KV offload path: extends the prefix cache into host DRAM, optionally backed by further tiers. Composes with the selected strategy.",
      "description": "vLLM's built-in OffloadingConnector extends the prefix cache beyond GPU memory: completed KV blocks are copied out by DMA alongside compute and promoted back on a hit. A CPU DRAM tier is always present and is the only tier with GPU access; optional secondary tiers sit behind it and stage their transfers through it. Pick the tiers below."
    }
  },
  "kv_offload": {
    "simple": {
      "display_name": "Simple (CPU offload)",
      "label": "Simple",
      "order": 1,
      "description": "Offload KV cache to CPU DRAM on each serving node via SimpleCPUOffloadConnector — one flag, no extra infrastructure. ~220 GiB of host RAM used per GPU rank.",
      "args": [
        "--kv-transfer-config",
        "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":236223201280,\"lazy_offload\":false}}"
      ],
      "pd": {
        "roles": [
          "prefill"
        ]
      }
    },
    "offloading_cpu": {
      "display_name": "CPU DRAM only",
      "label": "CPU",
      "group": "offloading",
      "order": 1,
      "requires_opt_in": true,
      "brands": [
        "NVIDIA",
        "AMD"
      ],
      "description": "Single CPU DRAM tier, under LRU eviction. cpu_bytes_to_use is the total across all workers — not per rank, as Simple's knob is — and should exceed the aggregate GPU KV cache, or the CPU tier just mirrors what the GPU already holds.",
      "args": [
        "--kv-transfer-config",
        "{\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":549755813888,\"blocks_per_chunk\":4}}"
      ]
    },
    "offloading_fs": {
      "display_name": "CPU DRAM + filesystem",
      "label": "CPU + Filesystem",
      "group": "offloading",
      "order": 2,
      "requires_opt_in": true,
      "brands": [
        "NVIDIA",
        "AMD"
      ],
      "description": "Adds a filesystem tier behind the CPU tier, so cached prefixes outlive DRAM capacity and survive a restart. The CPU tier becomes a smaller staging tier; the filesystem carries the capacity. Point root_dir at fast local NVMe, or at a shared mount to pool one cache across instances. Object-store (obj) and RDMA-peer (p2p) tiers configure the same way in secondary_tiers.",
      "env": {
        "PYTHONHASHSEED": "0"
      },
      "args": [
        "--kv-transfer-config",
        "{\"kv_connector\":\"OffloadingConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"spec_name\":\"TieringOffloadingSpec\",\"cpu_bytes_to_use\":34359738368,\"blocks_per_chunk\":4,\"eviction_policy\":\"lru\",\"secondary_tiers\":[{\"type\":\"fs\",\"root_dir\":\"/mnt/kv_cache\",\"n_read_threads\":32,\"n_write_threads\":16}]}}"
      ]
    },
    "lmcache": {
      "display_name": "LMCache",
      "order": 4,
      "description": "Offload KV cache to a shared CPU-DRAM pool via LMCacheMPConnector — prefix caching beyond GPU KV capacity, one pool per node (not per-rank segments). Adds an LMCache Server tab; run it in a separate terminal before vllm serve.",
      "strategies": [
        "single_node_tp",
        "single_node_tep",
        "single_node_dep"
      ],
      "args": [
        "--kv-transfer-config",
        "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.port\":5555,\"lmcache.mp.mq_timeout\":300}}"
      ],
      "install": "uv pip install \"lmcache==0.5.1\"",
      "docs_url": "https://docs.lmcache.ai/",
      "companion": {
        "label": "LMCache Server",
        "description": "Run on the same node in a separate terminal, before vllm serve — owns the shared CPU-DRAM KV pool.",
        "command": "# --port must match \"lmcache.mp.port\" in --kv-transfer-config.\n# Size --l1-size-gb to ~75% of the host DRAM you can dedicate — pinned\n# memory is unswappable; the pool grows lazily from --l1-init-size-gb.\n# --max-gpu-workers 1 avoids concurrent-GPU-transfer stalls under heavy\n# async-load pressure; CPU-side workers stay at 8.\nlmcache server \\\n  --port 5555 \\\n  --http-port 8080 \\\n  --l1-size-gb 512 \\\n  --l1-init-size-gb 20 \\\n  --max-gpu-workers 1 \\\n  --max-cpu-workers 8 \\\n  --chunk-size 1024 \\\n  --l1-align-bytes 16384 \\\n  --eviction-trigger-watermark 0.85 \\\n  --eviction-ratio 0.10 \\\n  --eviction-policy LRU \\\n  --supported-transfer-mode lmcache_driven\n"
      }
    }
  }
}