{
  "multi_node_dep": {
    "name": "multi_node_dep",
    "deploy_type": "multi_node",
    "display_name": "Multi-Node Data + Expert Parallel",
    "short_label": "Multi-node DEP",
    "orientation": "throughput",
    "description": "Multi-node DEP. Each node runs TEP over its local GPUs, with DP across nodes. Best throughput for MoE models across multiple nodes. Pass -dp <total_workers> at deploy time. Uses hybrid load balancing across DP workers.\n",
    "hardware_match": {
      "min_gpus": 2,
      "multi_node": true,
      "architecture": "moe"
    },
    "vllm_args": [
      "--enable-expert-parallel",
      "--data-parallel-hybrid-lb"
    ],
    "parallel_flag": "--data-parallel-size",
    "env": {
      "GLOO_SOCKET_IFNAME": "$IFACE_NAME",
      "NCCL_SOCKET_IFNAME": "$IFACE_NAME"
    }
  },
  "multi_node_tep": {
    "name": "multi_node_tep",
    "deploy_type": "multi_node",
    "display_name": "Multi-Node Tensor + Expert Parallel",
    "short_label": "Multi-node TEP",
    "orientation": "balanced",
    "description": "Multi-node TEP. Combines cross-node tensor parallel with expert parallel for MoE models. All GPUs across nodes participate in TP, with EP splitting expert layers. Requires InfiniBand interconnect.\n",
    "hardware_match": {
      "min_gpus": 2,
      "multi_node": true,
      "architecture": "moe"
    },
    "vllm_args": [
      "--enable-expert-parallel"
    ],
    "parallel_flag": "--tensor-parallel-size",
    "env": {
      "GLOO_SOCKET_IFNAME": "$IFACE_NAME",
      "NCCL_SOCKET_IFNAME": "$IFACE_NAME"
    }
  },
  "multi_node_tp": {
    "name": "multi_node_tp",
    "deploy_type": "multi_node",
    "display_name": "Multi-Node Tensor Parallel",
    "short_label": "Multi-node TP",
    "orientation": "latency",
    "description": "Multi-node TP. Splits the model across GPUs spanning multiple nodes. TP is set to total GPU count across all nodes. Head node serves requests, worker nodes run with --headless. Requires InfiniBand interconnect.\n",
    "hardware_match": {
      "min_gpus": 2,
      "multi_node": true
    },
    "parallel_flag": "--tensor-parallel-size",
    "env": {
      "GLOO_SOCKET_IFNAME": "$IFACE_NAME",
      "NCCL_SOCKET_IFNAME": "$IFACE_NAME"
    }
  },
  "multi_node_tp_dp": {
    "name": "multi_node_tp_dp",
    "deploy_type": "multi_node",
    "display_name": "Multi-Node TP + Data Parallel",
    "short_label": "TP+DP",
    "orientation": "balanced",
    "description": "Tensor parallel within each node, data parallel across nodes — one full model replica per node. TP stays inside the high-bandwidth intra-node NVLink domain; the inter-node link carries only DP coordination, no tensor traffic. E.g. 2× H200 nodes → TP8·DP2, 4× GB200/GB300 NVL4 trays → TP4·DP4.\n",
    "hardware_match": {
      "min_gpus": 2,
      "multi_node": true
    },
    "parallelism": "tp_dp",
    "vllm_args": [],
    "parallel_flag": "--tensor-parallel-size",
    "env": {
      "GLOO_SOCKET_IFNAME": "$IFACE_NAME",
      "NCCL_SOCKET_IFNAME": "$IFACE_NAME"
    }
  },
  "multi_node_tp_pp": {
    "name": "multi_node_tp_pp",
    "deploy_type": "multi_node",
    "display_name": "Multi-Node TP + Pipeline Parallel",
    "short_label": "TP+PP",
    "orientation": "throughput",
    "description": "Tensor parallel within each node, pipeline parallel across nodes. TP stays inside the high-bandwidth intra-node NVLink/IB domain; PP is what crosses the slower cross-node link. Good fit for very large models (e.g. Kimi-K2 1T on 16×H800) where pure TP across nodes would bottleneck on inter-node bandwidth.\n",
    "hardware_match": {
      "min_gpus": 2,
      "multi_node": true
    },
    "parallelism": "tp_pp",
    "vllm_args": [],
    "parallel_flag": "--tensor-parallel-size",
    "env": {
      "GLOO_SOCKET_IFNAME": "$IFACE_NAME",
      "NCCL_SOCKET_IFNAME": "$IFACE_NAME"
    }
  },
  "pd_cluster": {
    "name": "pd_cluster",
    "deploy_type": "pd_cluster",
    "display_name": "Prefill/Decode Disaggregation",
    "short_label": "PD",
    "orientation": "production",
    "description": "Disaggregated prefill/decode cluster. Splits inference into separate prefill and decode workers communicating via NIXL KV cache transfer. Optimal for high-concurrency, low-latency production deployments. Each role can be a single-node or multi-node DP/TP/EP pool, with its own HTTP port, NIXL side-channel port, and DP coordinator.\n",
    "hardware_match": {
      "min_gpus": 8,
      "multi_node": true
    },
    "prefill": {
      "vllm_args": [
        "--port",
        "8001"
      ],
      "env": {
        "NCCL_CUMEM_ENABLE": "1",
        "NCCL_MNNVL_ENABLE": "1",
        "NCCL_NVLS_ENABLE": "1",
        "UCX_NET_DEVICES": "all"
      }
    },
    "decode": {
      "vllm_args": [
        "--port",
        "8002",
        "--compilation-config",
        "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}"
      ]
    },
    "connectors": {
      "nixl": {
        "display_name": "NIXL",
        "default": true,
        "description": "NIXL pull transfer: decode reads each request's KV from prefill. Works with every PD router.",
        "docs": "https://docs.vllm.ai/en/latest/features/nixl_connector_usage.html",
        "routers": [
          "vllm",
          "dynamo",
          "smg"
        ],
        "prefill": {
          "config": {
            "kv_connector": "NixlConnector",
            "kv_role": "kv_producer",
            "kv_load_failure_policy": "fail"
          },
          "env": {
            "VLLM_NIXL_SIDE_CHANNEL_PORT": "5557",
            "VLLM_NIXL_SIDE_CHANNEL_HOST": "$PREFILL_NODE_1"
          }
        },
        "decode": {
          "config": {
            "kv_connector": "NixlConnector",
            "kv_role": "kv_consumer",
            "kv_load_failure_policy": "fail"
          },
          "env": {
            "VLLM_NIXL_SIDE_CHANNEL_PORT": "5558",
            "VLLM_NIXL_SIDE_CHANNEL_HOST": "$DECODE_NODE_1"
          }
        }
      },
      "mooncake": {
        "display_name": "Mooncake",
        "description": "Mooncake push transfer over RDMA: prefill writes KV into decode. The router queries each prefill's bootstrap server for its engines. Set WITH_NVIDIA_PEERMEM=0 on hosts without the nvidia-peermem module (GB200).",
        "docs": "https://docs.vllm.ai/en/latest/features/mooncake_connector_usage.html",
        "brands": [
          "NVIDIA"
        ],
        "prefill": {
          "config": {
            "kv_connector": "MooncakeConnector",
            "kv_role": "kv_producer",
            "kv_load_failure_policy": "fail"
          },
          "env": {
            "VLLM_MOONCAKE_BOOTSTRAP_PORT": "8998"
          }
        },
        "decode": {
          "config": {
            "kv_connector": "MooncakeConnector",
            "kv_role": "kv_consumer",
            "kv_load_failure_policy": "fail"
          }
        },
        "router": {
          "kv_connector": "mooncake",
          "bootstrap_port": 8998
        },
        "install": [
          {
            "note": "Mooncake transfer engine on every prefill / decode node (CUDA 12: mooncake-transfer-engine)",
            "command": "uv pip install mooncake-transfer-engine-cuda13"
          }
        ],
        "docker": {
          "host_network": true,
          "flags": [
            "--ulimit",
            "memlock=-1"
          ]
        }
      },
      "moriio": {
        "display_name": "MoRI-IO",
        "description": "ROCm MoRI-IO transfer over RDMA. Workers register with the router (proxy_ip / proxy_ping_port), which discovers them instead of listing endpoints; WRITE mode by default, READ with read_mode.",
        "docs": "https://docs.vllm.ai/en/latest/features/moriio_connector_usage.html",
        "brands": [
          "AMD"
        ],
        "prefill": {
          "config": {
            "kv_connector": "MoRIIOConnector",
            "kv_role": "kv_producer",
            "kv_load_failure_policy": "fail",
            "kv_connector_extra_config": {
              "proxy_ip": "$ROUTER_HOST",
              "proxy_ping_port": "36367",
              "http_port": "{port}",
              "handshake_port": "6301",
              "notify_port": "61005"
            }
          }
        },
        "decode": {
          "config": {
            "kv_connector": "MoRIIOConnector",
            "kv_role": "kv_consumer",
            "kv_load_failure_policy": "fail",
            "kv_connector_extra_config": {
              "proxy_ip": "$ROUTER_HOST",
              "proxy_ping_port": "36367",
              "http_port": "{port}",
              "handshake_port": "7301",
              "notify_port": "62005"
            }
          }
        },
        "router": {
          "kv_connector": "moriio",
          "discovery_port": 36367
        },
        "install": [
          {
            "note": "MoRI on every prefill / decode node (the ROCm image ships it)",
            "command": "uv pip install amd_mori"
          }
        ],
        "docker": {
          "host_network": true,
          "flags": [
            "--ulimit",
            "memlock=-1"
          ]
        }
      }
    },
    "parallel_flag": "--tensor-parallel-size",
    "router": {
      "policy": "round_robin"
    },
    "smg": {
      "display_name": "SMG",
      "description": "SMG HTTP router with native vLLM prefill/decode workers and explicit NIXL registration. Supports TP/TEP pools without KV offload in this recipe.\n",
      "docs": "https://lightseek.org/smg/getting-started/pd-disaggregation#vllm-pd-over-http",
      "version": "1.11.0",
      "install": "uv pip install \"smg==1.11.0\"",
      "docker_image": "lightseekorg/smg:1.11.0",
      "policy": "round_robin",
      "grpc": {
        "docs": "https://lightseek.org/smg/getting-started/pd-disaggregation#vllm-pd-over-grpc",
        "packages": "\"smg-grpc-servicer>=0.13.0\" \"smg-grpc-proto>=0.4.22\" \"protobuf<7\""
      }
    },
    "dynamo": {
      "display_name": "Dynamo",
      "description": "NVIDIA Dynamo frontend + dynamo.vllm workers. Workers discover each other through etcd (TCP request plane, no NATS) instead of fixed HTTP endpoints, and the frontend adds KV-aware / least-loaded routing. Same prefill/decode pools as the native vLLM router.\n",
      "docs": "https://docs.nvidia.com/dynamo/latest/backends/vllm/README.html",
      "brands": [
        "NVIDIA"
      ],
      "version": "1.5.0",
      "install": [
        {
          "note": "NVIDIA Dynamo runtime — on every prefill / decode node and the frontend host",
          "command": "uv pip install --extra-index-url https://pypi.nvidia.com \"ai-dynamo-runtime==1.5.0\" \"ai-dynamo==1.5.0\""
        },
        {
          "note": "Confirm the vLLM / Dynamo versions every node resolved",
          "optional": true,
          "command": "python3 -c 'import importlib.metadata as m; print({p: m.version(p) for p in (\"vllm\", \"ai-dynamo\", \"ai-dynamo-runtime\")})'"
        }
      ],
      "launcher": "python3 -m dynamo.vllm",
      "remove_args": [
        "--port"
      ],
      "env": {
        "ETCD_ENDPOINTS": "http://$DYN_CONTROL_IP:2379",
        "DYN_NAMESPACE": "vllm-pd",
        "DYN_REQUEST_PLANE": "tcp"
      },
      "prefill": {
        "vllm_args": [
          "--disaggregation-mode",
          "prefill",
          "--kv-transfer-config",
          "{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\"}"
        ]
      },
      "decode": {
        "vllm_args": [
          "--disaggregation-mode",
          "decode",
          "--kv-transfer-config",
          "{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_both\",\"kv_load_failure_policy\":\"fail\"}"
        ]
      },
      "infra": {
        "label": "etcd",
        "description": "Control plane — start once on $DYN_CONTROL_IP before any worker or the frontend.",
        "command": "docker run -d --name dynamo-etcd --network host quay.io/coreos/etcd:v3.5.21 \\\n  etcd --listen-client-urls http://0.0.0.0:2379 \\\n  --advertise-client-urls http://$DYN_CONTROL_IP:2379\n"
      },
      "frontend": {
        "label": "Frontend",
        "router_mode": "round-robin"
      }
    }
  },
  "single_node_dep": {
    "name": "single_node_dep",
    "deploy_type": "single_node",
    "display_name": "Data + Expert Parallel",
    "short_label": "DEP",
    "orientation": "throughput",
    "description": "Single-node DEP. Expert layers are shared across all GPUs via EP, dense layers run independently in DP groups. Best throughput for MoE models on 8-GPU single nodes. For MoE models only.\n",
    "hardware_match": {
      "min_gpus": 2,
      "max_gpus": 8,
      "multi_node": false,
      "architecture": "moe"
    },
    "vllm_args": [
      "--enable-expert-parallel",
      "--port",
      "8100"
    ],
    "parallel_flag": "--data-parallel-size",
    "companion": {
      "label": "vLLM Router",
      "description": "Routes client requests across the DP ranks of the single engine process. --intra-node-data-parallel-size must match the engine's --data-parallel-size; consistent_hash keeps a conversation pinned to the rank that already holds its prefix.",
      "install": "uv pip install vllm-router",
      "start": "after",
      "command": "vllm-router \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --policy consistent_hash \\\n  --intra-node-data-parallel-size {dp_size} \\\n  --worker-urls http://localhost:{port} \\\n  --worker-startup-timeout-secs 1800\n"
    }
  },
  "single_node_dpa_tp": {
    "name": "single_node_dpa_tp",
    "deploy_type": "single_node",
    "display_name": "DP Attention + Tensor Parallel",
    "short_label": "DPA+TP",
    "orientation": "throughput",
    "description": "Single-node data-parallel attention with tensor-sharded experts (DPA+TP). Every GPU runs its own attention replica with a private KV cache, while the MoE expert weights are tensor-sharded across all local GPUs instead of replicated per replica. Aggregate KV cache capacity scales with the GPU count, so long multi-turn conversations stay resident instead of being evicted and re-prefilled. A vllm-router in front spreads requests across the DP ranks. For MoE models only.\n",
    "hardware_match": {
      "min_gpus": 2,
      "max_gpus": 8,
      "multi_node": false,
      "architecture": "moe"
    },
    "parallelism": "dpa_tp",
    "remove_args": [
      "--enable-expert-parallel"
    ],
    "vllm_args": [
      "--port",
      "8100"
    ],
    "parallel_flag": "--data-parallel-size",
    "kv_offload": false,
    "companion": {
      "label": "vLLM Router",
      "description": "Routes client requests across the DP ranks of the single engine process. --intra-node-data-parallel-size must match the engine's --data-parallel-size; consistent_hash keeps a conversation pinned to the rank that already holds its prefix.",
      "install": "uv pip install vllm-router",
      "start": "after",
      "command": "vllm-router \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --policy consistent_hash \\\n  --intra-node-data-parallel-size {dp_size} \\\n  --worker-urls http://localhost:{port} \\\n  --worker-startup-timeout-secs 1800\n"
    }
  },
  "single_node_tep": {
    "name": "single_node_tep",
    "deploy_type": "single_node",
    "display_name": "Tensor + Expert Parallel",
    "short_label": "TEP",
    "orientation": "balanced",
    "description": "Single-node TEP. TP splits dense layers and EP splits expert layers across local GPUs. TP must be set to GPU count to avoid OOM from replicated dense layers. For MoE models only.\n",
    "hardware_match": {
      "min_gpus": 2,
      "max_gpus": 8,
      "multi_node": false,
      "architecture": "moe"
    },
    "vllm_args": [
      "--enable-expert-parallel"
    ],
    "parallel_flag": "--tensor-parallel-size"
  },
  "single_node_tp": {
    "name": "single_node_tp",
    "deploy_type": "single_node",
    "display_name": "Tensor Parallel",
    "short_label": "TP",
    "orientation": "latency",
    "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n",
    "hardware_match": {
      "min_gpus": 1,
      "max_gpus": 8,
      "multi_node": false
    },
    "vllm_args": [],
    "parallel_flag": "--tensor-parallel-size"
  }
}