{ "hf_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "meta": { "title": "Qwen3-Next-80B-A3B-Instruct", "slug": "qwen3-next-80b-a3b-instruct", "provider": "Qwen", "description": "Advanced Qwen3-Next MoE model (80B total / 3B active) with hybrid attention, highly sparse experts, and multi-token prediction.", "date_added": "2025-09-11", "date_updated": "2026-09-10", "tasks": [ "text" ], "performance_headline": "Highly sparse MoE with MTP-accelerated decoding, runs on 4x H200/H20/A100/A800", "related_recipes": [], "hardware": { "h200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve Qwen/Qwen3-Next-80B-A3B-Instruct \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes", "argv": [ "vllm", "serve", "Qwen/Qwen3-Next-80B-A3B-Instruct", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "hermes" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest Qwen/Qwen3-Next-80B-A3B-Instruct \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "Qwen/Qwen3-Next-80B-A3B-Instruct", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "hermes" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy โ€” works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e ยท 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/Qwen/Qwen3-Next-80B-A3B-Instruct/strategies/single_node_tep.json", "single_node_dep": "/Qwen/Qwen3-Next-80B-A3B-Instruct/strategies/single_node_dep.json", "multi_node_tp": "/Qwen/Qwen3-Next-80B-A3B-Instruct/strategies/multi_node_tp.json", "multi_node_tp_pp": "/Qwen/Qwen3-Next-80B-A3B-Instruct/strategies/multi_node_tp_pp.json", "multi_node_dep": "/Qwen/Qwen3-Next-80B-A3B-Instruct/strategies/multi_node_dep.json", "multi_node_tep": "/Qwen/Qwen3-Next-80B-A3B-Instruct/strategies/multi_node_tep.json", "kv_store_centralized_mooncake": "/Qwen/Qwen3-Next-80B-A3B-Instruct/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/Qwen/Qwen3-Next-80B-A3B-Instruct/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/h200.json", "h100": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/h100.json", "b200": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/b200.json", "gb200": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/gb200.json", "b300": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/b300.json", "gb300": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/gb300.json", "mi300x": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/mi300x.json", "mi325x": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/mi325x.json", "mi350x": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/mi350x.json", "mi355x": "/Qwen/Qwen3-Next-80B-A3B-Instruct/hw/mi355x.json" } }, "model": { "model_id": "Qwen/Qwen3-Next-80B-A3B-Instruct", "min_vllm_version": "0.17.0", "architecture": "moe", "parameter_count": "80B", "active_parameters": "3B", "context_length": 262144, "base_args": [], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable automatic tool choice with Hermes parser", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "hermes" ] }, "spec_decoding": { "description": "Multi-token prediction speculative decoding for lower latency", "args": [ "--speculative-config", "{\"method\":\"qwen3_next_mtp\",\"num_speculative_tokens\":2}", "--no-enable-chunked-prefill" ] }, "long_context": { "description": "YaRN RoPE scaling to the model card's validated 1M-token context (factor 4.0 over the native 262,144). Static YaRN holds the scaling factor constant regardless of input length, so leave this off unless you actually serve long prompts.", "args": [ "--hf-overrides", "{\"rope_parameters\":{\"rope_type\":\"yarn\",\"rope_theta\":10000000,\"partial_rotary_factor\":0.25,\"factor\":4.0,\"original_max_position_embeddings\":262144}}", "--max-model-len", "1010000" ], "env": { "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1" } } }, "opt_in_features": [ "spec_decoding", "long_context" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 192, "description": "Full precision BF16 โ€” fits on 4x H200/H20/A100/A800" }, "fp8": { "model_id": "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8", "precision": "fp8", "vram_minimum_gb": 96, "description": "Qwen official FP8 checkpoint โ€” recommended on SM90/SM100", "json": "/Qwen/Qwen3-Next-80B-A3B-Instruct-FP8.json" }, "nvfp4": { "model_id": "nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 48, "description": "NVIDIA NVFP4 quantized weights for Blackwell GPUs", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/nvidia/Qwen3-Next-80B-A3B-Instruct-NVFP4.json" }, "redhat_fp8": { "model_id": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic", "precision": "fp8", "label": "FP8 Dynamic (RedHat)", "vram_minimum_gb": 96, "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-dynamic.json" }, "redhat_fp8_plain": { "model_id": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8", "precision": "fp8", "label": "FP8 (RedHat)", "vram_minimum_gb": 96, "description": "RedHatAI FP8 quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8.json" }, "redhat_fp8_block": { "model_id": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block", "precision": "fp8", "label": "FP8 Block (RedHat)", "vram_minimum_gb": 96, "description": "RedHatAI FP8 block quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Qwen3-Next-80B-A3B-Instruct-FP8-block.json" }, "redhat_nvfp4": { "model_id": "RedHatAI/Qwen3-Next-80B-A3B-Instruct-NVFP4", "precision": "nvfp4", "label": "NVFP4 (RedHat)", "vram_minimum_gb": 48, "description": "RedHatAI NVFP4 quantization", "json": "/RedHatAI/Qwen3-Next-80B-A3B-Instruct-NVFP4.json" } }, "hardware_overrides": { "blackwell": { "extra_args": [ "--moe-backend", "flashinfer_trtllm", "--attention-backend", "FLASH_ATTN" ], "extra_env": { "VLLM_USE_DEEP_GEMM": "0" } } }, "guide": "## Overview\n\n[Qwen3-Next](https://qwen.ai/blog?id=4074cca80393150c248e508aa62983f9cb7d27cd&from=research.latest-advancements-list) is an advanced LLM from the Qwen team featuring:\n- A hybrid attention mechanism\n- A highly sparse Mixture-of-Experts (MoE) structure\n- Training-stability-friendly optimizations\n- A multi-token prediction mechanism for faster inference\n\n## Prerequisites\n\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto\n```\n\n## Deployment Configurations\n\nLaunch on 4x H200/H20 or 4x A100/A800 GPUs.\n\n### Basic Multi-GPU (BF16)\n```bash\nvllm serve Qwen/Qwen3-Next-80B-A3B-Instruct \\\n --tensor-parallel-size 4 \\\n --served-model-name qwen3-next \\\n --enable-prefix-caching\n```\n\nIf you hit `torch.AcceleratorError: CUDA error: an illegal memory access was encountered`, add `--compilation_config.cudagraph_mode=PIECEWISE`.\n\n### FP8 (SM90/SM100)\n```bash\nvllm serve Qwen/Qwen3-Next-80B-A3B-Instruct-FP8 \\\n --tensor-parallel-size 4 \\\n --enable-prefix-caching\n```\n\nOn SM100, accelerate with the FlashInfer TRTLLM MoE kernel:\n```bash\nVLLM_USE_DEEP_GEMM=0 \\\nvllm serve Qwen/Qwen3-Next-80B-A3B-Instruct-FP8 \\\n --tensor-parallel-size 4 \\\n --moe-backend flashinfer_trtllm \\\n --attention-backend FLASH_ATTN\n```\n\n### MTP (Multi-Token Prediction)\n```bash\nvllm serve Qwen/Qwen3-Next-80B-A3B-Instruct \\\n --tokenizer-mode auto --gpu-memory-utilization 0.8 \\\n --speculative-config '{\"method\": \"qwen3_next_mtp\", \"num_speculative_tokens\": 2}' \\\n --tensor-parallel-size 4 --no-enable-chunked-prefill\n```\n\n### Tool / Function Calling\n```bash\nvllm serve ... --tool-call-parser hermes --enable-auto-tool-choice\n```\n\n### AMD (MI300X/MI325X/MI355X)\n```bash\nuv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/0.14.1/rocm700\n```\n```bash\nSAFETENSORS_FAST_GPU=1 \\\nVLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \\\nvllm serve Qwen/Qwen3-Next-80B-A3B-Instruct \\\n --tensor-parallel-size 4 \\\n --max-model-len 32768 \\\n --no-enable-prefix-caching \\\n --trust-remote-code\n```\n\n## Client Usage\n\nBenchmark:\n```bash\nvllm bench serve \\\n --backend vllm \\\n --model Qwen/Qwen3-Next-80B-A3B-Instruct \\\n --served-model-name qwen3-next \\\n --endpoint /v1/completions \\\n --dataset-name random \\\n --random-input 2048 \\\n --random-output 1024 \\\n --max-concurrency 10 \\\n --num-prompt 100\n```\n\n## Troubleshooting\n\n- **Sub-optimal MoE performance warning**: Tune the MoE Triton kernel with [benchmark_moe](https://github.com/vllm-project/vllm/blob/main/benchmarks/kernels/benchmark_moe.py), then set `VLLM_TUNED_CONFIG_FOLDER` to the directory containing the generated config.\n- **IMA error in DP mode**: add `--compilation_config.cudagraph_mode=PIECEWISE`.\n- For more parallel topologies, see the [Data Parallel Deployment docs](https://docs.vllm.ai/en/latest/serving/data_parallel_deployment.html).\n\n## References\n\n- [Model card](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct)\n- [FP8 checkpoint](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct-FP8)\n- [Qwen3-Next blog](https://qwen.ai/blog?id=4074cca80393150c248e508aa62983f9cb7d27cd)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/Qwen/Qwen3-Next-80B-A3B-Instruct.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Qwen/Qwen3-Next-80B-A3B-Instruct.yaml", "overlays": [] } }