{ "hf_id": "Qwen/Qwen3.5-397B-A17B", "meta": { "title": "Qwen3.5-397B", "slug": "qwen3.5-397b", "provider": "Qwen", "description": "Multimodal MoE model with gated delta networks architecture, 397B total / 17B active parameters, up to 262K context", "date_added": "2026-02-16", "date_updated": "2026-10-07", "tasks": [ "multimodal", "text" ], "performance_headline": "Verified on 8x H200, 8x MI300X/MI355X, a single MI350X/MI355X (MXFP4), GB200 nodes, and Ascend 950DT", "related_recipes": [ "Qwen/Qwen3.6-35B-A3B" ], "hardware": { "h200": "verified", "gb200": "verified", "mi300x": "verified", "mi325x": "verified", "mi350x": "verified", "mi355x": "verified", "ascend_950dt": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0", "VLLM_USE_RUST_FRONTEND": "1" }, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve Qwen/Qwen3.5-397B-A17B \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser qwen3 \\\n --mm-encoder-tp-mode data", "argv": [ "vllm", "serve", "Qwen/Qwen3.5-397B-A17B", "--trust-remote-code", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "qwen3", "--mm-encoder-tp-mode", "data" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_DEEP_GEMM_WARMUP=skip \\\n -e VLLM_USE_DEEP_GEMM=0 \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n vllm/vllm-openai:latest Qwen/Qwen3.5-397B-A17B \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser qwen3 \\\n --mm-encoder-tp-mode data", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_DEEP_GEMM_WARMUP=skip", "-e", "VLLM_USE_DEEP_GEMM=0", "-e", "VLLM_USE_RUST_FRONTEND=1", "vllm/vllm-openai:latest", "Qwen/Qwen3.5-397B-A17B", "--trust-remote-code", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "qwen3", "--mm-encoder-tp-mode", "data" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/Qwen/Qwen3.5-397B-A17B/strategies/single_node_tep.json", "single_node_dep": "/Qwen/Qwen3.5-397B-A17B/strategies/single_node_dep.json", "multi_node_tp": "/Qwen/Qwen3.5-397B-A17B/strategies/multi_node_tp.json", "multi_node_tp_pp": "/Qwen/Qwen3.5-397B-A17B/strategies/multi_node_tp_pp.json", "multi_node_dep": "/Qwen/Qwen3.5-397B-A17B/strategies/multi_node_dep.json", "multi_node_tep": "/Qwen/Qwen3.5-397B-A17B/strategies/multi_node_tep.json", "pd_cluster": "/Qwen/Qwen3.5-397B-A17B/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/Qwen/Qwen3.5-397B-A17B/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/Qwen/Qwen3.5-397B-A17B/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/Qwen/Qwen3.5-397B-A17B/hw/h200.json", "h100": "/Qwen/Qwen3.5-397B-A17B/hw/h100.json", "b200": "/Qwen/Qwen3.5-397B-A17B/hw/b200.json", "gb200": "/Qwen/Qwen3.5-397B-A17B/hw/gb200.json", "b300": "/Qwen/Qwen3.5-397B-A17B/hw/b300.json", "gb300": "/Qwen/Qwen3.5-397B-A17B/hw/gb300.json", "mi300x": "/Qwen/Qwen3.5-397B-A17B/hw/mi300x.json", "mi325x": "/Qwen/Qwen3.5-397B-A17B/hw/mi325x.json", "mi350x": "/Qwen/Qwen3.5-397B-A17B/hw/mi350x.json", "mi355x": "/Qwen/Qwen3.5-397B-A17B/hw/mi355x.json", "ascend_950dt": "/Qwen/Qwen3.5-397B-A17B/hw/ascend_950dt.json" } }, "model": { "model_id": "Qwen/Qwen3.5-397B-A17B", "min_vllm_version": "0.17.0", "default_frontend": "rust", "architecture": "moe", "parameter_count": "397B", "active_parameters": "17B", "context_length": 262144, "base_args": [ "--trust-remote-code" ], "base_env": { "VLLM_DEEP_GEMM_WARMUP": "skip", "VLLM_USE_DEEP_GEMM": "0" }, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable automatic tool choice with Qwen3 Coder parser", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder" ] }, "reasoning": { "description": "Enable chain-of-thought reasoning with Qwen3 parser", "args": [ "--reasoning-parser", "qwen3" ] }, "spec_decoding": { "description": "Multiple options for multi-token prediction speculative decoding for lower latency. Also has DFlash option. ", "default_mode": "mtp", "modes": { "mtp": { "description": "Native MTP head, 3 draft tokens.", "variants": [ "default", "nvfp4", "fp8", "gptq_int4", "amd_mxfp4" ], "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" ] }, "dflash": { "label": "DFlash", "description": "DFlash speculative decoding with an externally trained draft model; the served checkpoint is unchanged.", "variants": [ "default" ], "args": [ "--speculative-config", "{\"model\": \"RedHatAI/Qwen3.5-397B-A17B-speculator.dflash\", \"num_speculative_tokens\": 7, \"method\": \"dflash\"}" ] }, "qwen3_5_mtp": { "label": "MTP (Ascend)", "description": "MTP mode documented for vLLM Ascend.", "variants": [ "ascend_w8a8_mtp", "ascend_w4a4_mxfp4" ], "args": [ "--speculative-config", "{\"method\":\"qwen3_5_mtp\",\"num_speculative_tokens\":3}" ] } } }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "encoder_parallel": { "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.", "args": [ "--mm-encoder-tp-mode", "data" ] }, "long_context": { "description": "YaRN RoPE scaling past the native 262,144-token window to ~1,010,000 tokens. Static YaRN holds the scaling factor constant regardless of input length, so leave this off unless you actually serve long prompts; for ~524K, halve the factor to 2.0.", "args": [ "--hf-overrides", "{\"text_config\": {\"rope_parameters\": {\"mrope_interleaved\": true, \"mrope_section\": [11, 11, 10], \"rope_type\": \"yarn\", \"rope_theta\": 10000000, \"partial_rotary_factor\": 0.25, \"factor\": 4.0, \"original_max_position_embeddings\": 262144}}}", "--max-model-len", "1010000" ], "env": { "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1" } } }, "opt_in_features": [ "spec_decoding", "text_only", "long_context" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 953, "description": "Full precision BF16 — requires 8x H200 or equivalent" }, "nvfp4": { "model_id": "nvidia/Qwen3.5-397B-A17B-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 238, "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/nvidia/Qwen3.5-397B-A17B-NVFP4.json" }, "fp8": { "model_id": "RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic", "precision": "fp8", "vram_minimum_gb": 477, "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Qwen3.5-397B-A17B-FP8-dynamic.json" }, "gptq_int4": { "model_id": "Qwen/Qwen3.5-397B-A17B-GPTQ-Int4", "precision": "int4", "vram_minimum_gb": 239, "description": "GPTQ Int4 checkpoint — halves VRAM vs FP8", "json": "/Qwen/Qwen3.5-397B-A17B-GPTQ-Int4.json" }, "amd_mxfp4": { "model_id": "amd/Qwen3.5-397B-A17B-MXFP4", "precision": "mxfp4", "label": "MXFP4 (W4A4)", "vram_minimum_gb": 287, "description": "AMD Quark MXFP4 checkpoint for MI350X/MI355X (gfx950) — fits one 288 GB GPU", "supported_hardware": [ "arch:gfx950" ], "tp": 1, "min_vllm_version": "0.31.0", "extra_env": { "VLLM_ROCM_USE_AITER": "1" }, "extra_args": [ "--attention-backend", "ROCM_AITER_FA", "--max-num-seqs", "64" ], "json": "/amd/Qwen3.5-397B-A17B-MXFP4.json" }, "ascend_w8a8_mtp": { "model_id": "Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp", "precision": "int8", "label": "W8A8 (Ascend)", "vram_minimum_gb": 477, "supported_hardware": [ "ascend_950dt" ], "tp": 8, "min_vllm_version": "0.23.0", "default_modes": { "spec_decoding": "qwen3_5_mtp" }, "extra_args": [ "--distributed-executor-backend", "mp", "--data-parallel-size", "1", "--quantization", "ascend", "--served-model-name", "qwen3.5", "--max-num-seqs", "128", "--max-model-len", "133000", "--max-num-batched-tokens", "8192", "--gpu-memory-utilization", "0.95", "--enable-prefix-caching", "--async-scheduling", "--compilation-config", "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}", "--additional-config", "{\"enable_cpu_binding\":true}" ], "extra_env": { "VLLM_USE_MODELSCOPE": "True", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_BUFFSIZE": "400", "OMP_PROC_BIND": "false", "OMP_NUM_THREADS": "100", "TASK_QUEUE_ENABLE": "1", "VLLM_ASCEND_ENABLE_PREFETCH_MLP": "1", "HCCL_INTRA_PCIE_ENABLE": "1", "HCCL_INTRA_ROCE_ENABLE": "0" }, "hardware_overrides": { "ascend_950dt": { "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-950DT" } }, "json": "/Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp.json" }, "ascend_w4a4_mxfp4": { "model_id": "Eco-Tech/Qwen3.5-397B-A17B-w4a4-mxfp4", "precision": "mxfp4", "label": "W4A4 (Ascend)", "vram_minimum_gb": 239, "supported_hardware": [ "ascend_950dt" ], "tp": 8, "min_vllm_version": "0.23.0", "default_modes": { "spec_decoding": "qwen3_5_mtp" }, "extra_args": [ "--distributed-executor-backend", "mp", "--data-parallel-size", "1", "--quantization", "ascend", "--served-model-name", "qwen3.5", "--max-num-seqs", "128", "--max-model-len", "133000", "--max-num-batched-tokens", "8192", "--gpu-memory-utilization", "0.95", "--enable-prefix-caching", "--async-scheduling", "--compilation-config", "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}", "--additional-config", "{\"enable_cpu_binding\":true}" ], "extra_env": { "VLLM_USE_MODELSCOPE": "True", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_BUFFSIZE": "400", "OMP_PROC_BIND": "false", "OMP_NUM_THREADS": "100", "TASK_QUEUE_ENABLE": "1", "VLLM_ASCEND_ENABLE_PREFETCH_MLP": "1", "HCCL_INTRA_PCIE_ENABLE": "1", "HCCL_INTRA_ROCE_ENABLE": "0" }, "hardware_overrides": { "ascend_950dt": { "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-950DT" } }, "json": "/Eco-Tech/Qwen3.5-397B-A17B-w4a4-mxfp4.json" } }, "hardware_overrides": { "blackwell": { "extra_args": [ "--moe-backend", "flashinfer_trtllm" ] } }, "guide": "## Overview\n\n[Qwen3.5](https://huggingface.co/Qwen/Qwen3.5-397B-A17B) is a multimodal mixture-of-experts model featuring a gated delta networks architecture with 397B total parameters and 17B active parameters. This guide covers how to efficiently deploy and serve the model across different hardware configurations and workload profiles using vLLM.\n\n## Prerequisites\n\n### Pip Install\n\n#### NVIDIA\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend=auto\n```\n\n#### AMD\n> Note: The vLLM wheel for ROCm requires Python 3.12, ROCm 7.0, and glibc >= 2.35. If your environment does not meet these requirements, please use the Docker-based setup. Supported GPUs: MI300X, MI325X, MI355X.\n```bash\nuv venv --python 3.12\nsource .venv/bin/activate\nuv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm\n```\n\n### Docker\n\n#### NVIDIA\n```bash\ndocker run --gpus all \\\n -p 8000:8000 \\\n --ipc=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai Qwen/Qwen3.5-397B-A17B \\\n --tensor-parallel-size 8 \\\n --reasoning-parser qwen3 \\\n --enable-prefix-caching\n```\nFor Blackwell GPUs, use `vllm/vllm-openai:cu130-nightly`.\n\n#### AMD\n```bash\ndocker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined \\\n --group-add video \\\n --ipc=host \\\n -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-rocm:latest \\\n Qwen/Qwen3.5-397B-A17B-FP8 \\\n --tensor-parallel-size 8 \\\n --reasoning-parser qwen3 \\\n --enable-prefix-caching\n```\n\n## Deployment Configurations\n\nThe configurations below have been verified on 8x H200 GPUs and 8x MI300X/MI355X GPUs. We recommend using the official FP8 checkpoint [Qwen/Qwen3.5-397B-A17B-FP8](https://huggingface.co/Qwen/Qwen3.5-397B-A17B-FP8) for optimal serving efficiency.\n\n### Throughput-Focused (Text-Only)\n\nFor maximum text throughput under high concurrency, use `--language-model-only` to skip loading the vision encoder and free up memory for KV cache, and enable Expert Parallelism.\n\n```bash\nvllm serve Qwen/Qwen3.5-397B-A17B-FP8 \\\n -dp 8 \\\n --enable-expert-parallel \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-prefix-caching\n```\n\n### Throughput-Focused (Multimodal)\n\nFor multimodal workloads, use `--mm-encoder-tp-mode data` for data-parallel vision encoding and `--mm-processor-cache-type shm` for shared-memory caching of preprocessed multimodal inputs.\n\n```bash\nvllm serve Qwen/Qwen3.5-397B-A17B-FP8 \\\n -dp 8 \\\n --enable-expert-parallel \\\n --mm-encoder-tp-mode data \\\n --mm-processor-cache-type shm \\\n --reasoning-parser qwen3 \\\n --enable-prefix-caching\n```\n\nTo enable tool calling, add `--enable-auto-tool-choice --tool-call-parser qwen3_coder` to the serve command.\n\n### Latency-Focused\n\nFor latency-sensitive workloads at low concurrency, enable MTP-1 speculative decoding and disable prefix caching. MTP-1 reduces time-per-output-token (TPOT) with a high acceptance rate, at the cost of lower throughput under load.\n\n> Note (AMD): MTP runs on ROCm with vLLM 0.31.0 (verified with the single-GPU MXFP4 setup below). On ROCm,\n> pair it with `--kv-cache-dtype fp8`: with the default KV cache its speedup fades as the prompt grows.\n\n```bash\nvllm serve Qwen/Qwen3.5-397B-A17B-FP8 \\\n --tensor-parallel-size 8 \\\n --speculative-config '{\"method\": \"mtp\", \"num_speculative_tokens\": 1}' \\\n --reasoning-parser qwen3\n```\n\n### GB200 Deployment\n\nWe recommend using the NVFP4 checkpoint [nvidia/Qwen3.5-397B-A17B-NVFP4](https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4) for optimal serving efficiency on GB200 nodes.\n\n```bash\nvllm serve nvidia/Qwen3.5-397B-A17B-NVFP4 \\\n -dp 4 \\\n --enable-expert-parallel \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-prefix-caching\n```\n\n### MI355X Deployment\n\n```bash\nvllm serve Qwen/Qwen3.5-397B-A17B-FP8 \\\n -tp 2 \\\n --enable-expert-parallel \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-prefix-caching\n```\n\n### Single MI350X / MI355X (MXFP4)\n\nAMD's MXFP4 checkpoint [amd/Qwen3.5-397B-A17B-MXFP4](https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4) (239 GB)\nfits one 288 GB MI350X or MI355X: pick the **MXFP4 (W4A4)** variant above. It sets three things vLLM does not\npick on its own: `VLLM_ROCM_USE_AITER=1` turns on the AITER kernels; `--attention-backend ROCM_AITER_FA` replaces\nthe auto-selected `ROCM_ATTN`, whose decode slows down as the context grows on this hybrid model; and\n`--max-num-seqs 64`, because the default (1024) is more than the ~296 Mamba cache blocks a single GPU leaves,\nand the server would refuse to start.\n\n```bash\nVLLM_ROCM_USE_AITER=1 vllm serve amd/Qwen3.5-397B-A17B-MXFP4 \\\n --tensor-parallel-size 1 \\\n --attention-backend ROCM_AITER_FA \\\n --max-num-seqs 64 \\\n --trust-remote-code \\\n --reasoning-parser qwen3\n```\n\nSingle-stream decode in tokens/s at ~200 / ~2k / ~12k / ~30k prompt tokens (128 output tokens), one GPU,\n`vllm/vllm-openai-rocm:v0.31.0`:\n\n| | ~200 | ~2k | ~12k | ~30k |\n|---|---|---|---|---|\n| MXFP4, this variant (MI355X) | 119 | 119 | 119 | 117 |\n| + MTP (3 tokens) + `--kv-cache-dtype fp8` (MI355X) | 219 | 201 | 209 | 188 |\n| + MTP (3 tokens), default KV cache (MI350X) | 198 | 164 | 125 | 80 |\n| auto-selected `ROCM_ATTN` instead of `ROCM_AITER_FA` (MI355X) | 116 | 89 | 39 | 19 |\n| GPTQ-Int4, AITER + `ROCM_AITER_FA` (MI350X) | 75 | 74 | 74 | 74 |\n\nWith MTP, add `--kv-cache-dtype fp8`: on ROCm, MTP's gain otherwise fades as the prompt grows. The MXFP4\ncheckpoint quantizes activations as well as weights (W4A4): on the same wikitext-2 tokens vLLM scored PPL 3.43\nwith it and 3.39 with GPTQ-Int4 (W4A16), so GPTQ-Int4 is the higher-fidelity option at about two thirds of the speed.\n\n## Processing Ultra-Long Texts\n\nQwen3.5-397B-A17B natively supports `262,144` tokens. To serve longer inputs, turn on the\n**Long Context** feature above: it applies the model card's YaRN RoPE\nconfiguration via `--hf-overrides` and raises `--max-model-len` to ~1,010,000 tokens.\nStatic YaRN holds the scaling factor constant regardless of input length, so\nleave it off unless you actually serve long prompts; for a ~524K workload,\nhalve the card's `factor` to `2.0`.\n\nSee the [model card](https://huggingface.co/Qwen/Qwen3.5-397B-A17B#processing-ultra-long-texts) for the full parameter reference.\n\n## Client Usage\n\n```python\nimport time\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=\"EMPTY\",\n base_url=\"http://localhost:8000/v1\",\n timeout=3600\n)\n\nmessages = [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": \"https://ofasys-multimodal-wlcb-3-toshanghai.oss-accelerate.aliyuncs.com/wpf272043/keepme/image/receipt.png\"\n }\n },\n {\n \"type\": \"text\",\n \"text\": \"Read all the text in the image.\"\n }\n ]\n }\n]\n\nstart = time.time()\nresponse = client.chat.completions.create(\n model=\"Qwen/Qwen3.5-397B-A17B\",\n messages=messages,\n max_tokens=2048\n)\nprint(f\"Response costs: {time.time() - start:.2f}s\")\nprint(f\"Generated text: {response.choices[0].message.content}\")\n```\n\n## Troubleshooting\n\n**CUDA graph / Mamba cache size error**\n\nYou may encounter:\n```\nassert num_cache_lines >= batch\n```\nThis occurs because the CUDA graph capture size is larger than the Mamba cache size. Reduce `--max-cudagraph-capture-size` (default is 512). See https://github.com/vllm-project/vllm/pull/34571 for details.\n\n**Configuration tips**\n\n- **Disable Reasoning**: Add `--reasoning-parser qwen3 --default-chat-template-kwargs '{\"enable_thinking\": false}'` to disable reasoning mode via command-line parameters.\n- **Prefix Caching**: Prefix caching for Mamba cache \"align\" mode is currently experimental.\n- **Multi-token Prediction**: MTP-1 reduces per-token latency but degrades throughput under high concurrency because speculative tokens consume KV cache capacity. Adjust `num_speculative_tokens` (1-5) based on your use case.\n- **Encoder Data Parallelism**: `--mm-encoder-tp-mode data` deploys the vision encoder in a data-parallel fashion. This consumes additional memory and may require adjustment of `--gpu-memory-utilization`.\n## References\n\n- Model card: https://huggingface.co/Qwen/Qwen3.5-397B-A17B\n- FP8 checkpoint: https://huggingface.co/Qwen/Qwen3.5-397B-A17B-FP8\n- NVFP4 checkpoint: https://huggingface.co/nvidia/Qwen3.5-397B-A17B-NVFP4\n- GPTQ-Int4 checkpoint: https://huggingface.co/Qwen/Qwen3.5-397B-A17B-GPTQ-Int4\n- AMD MXFP4 checkpoint: https://huggingface.co/amd/Qwen3.5-397B-A17B-MXFP4\n- Ascend W8A8 checkpoint: https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-397B-A17B-w8a8-mtp\n- Ascend W4A4 checkpoint: https://www.modelscope.cn/models/Eco-Tech/Qwen3.5-397B-A17B-w4a4-mxfp4\n- vLLM Ascend tutorial: https://docs.vllm.ai/projects/ascend/en/latest/tutorials/models/Qwen3.5-397B-A17B.html\n- vLLM documentation: https://docs.vllm.ai/\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/Qwen/Qwen3.5-397B-A17B.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Qwen/Qwen3.5-397B-A17B.yaml", "overlays": [ { "source": "vllm-ascend", "repo": "vllm-project/vllm-ascend", "ref": "main", "sha": "02ac27a19cb7676304a9ca6985ba56ce29664536", "path": "recipes/overlays/Qwen/Qwen3.5-397B-A17B.yaml", "edit_url": "https://github.com/vllm-project/vllm-ascend/edit/main/recipes/overlays/Qwen/Qwen3.5-397B-A17B.yaml" } ] } }