{ "hf_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "meta": { "title": "NVIDIA Nemotron-3-Nano-30B-A3B", "slug": "nemotron-3-nano-30b-a3b", "provider": "NVIDIA", "description": "NVIDIA Nemotron-3-Nano Mamba-hybrid MoE (30B total / ~3B active) with BF16 and FP8 variants", "date_added": "2026-01-08", "date_updated": "2026-09-10", "tasks": [ "text" ], "related_recipes": [ "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16" ], "hardware": { "h100": "verified", "h200": "verified", "arc_pro_b60": "verified", "arc_pro_b70": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \\\n --trust-remote-code \\\n --async-scheduling \\\n --kv-cache-dtype auto \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser-plugin nano_v3_reasoning_parser.py \\\n --reasoning-parser nano_v3", "argv": [ "vllm", "serve", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "--trust-remote-code", "--async-scheduling", "--kv-cache-dtype", "auto", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser-plugin", "nano_v3_reasoning_parser.py", "--reasoning-parser", "nano_v3" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \\\n --trust-remote-code \\\n --async-scheduling \\\n --kv-cache-dtype auto \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser-plugin nano_v3_reasoning_parser.py \\\n --reasoning-parser nano_v3", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "--trust-remote-code", "--async-scheduling", "--kv-cache-dtype", "auto", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser-plugin", "nano_v3_reasoning_parser.py", "--reasoning-parser", "nano_v3" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "multi_node_tp": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/strategies/multi_node_tp.json", "multi_node_tp_pp": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/strategies/multi_node_tp_pp.json", "multi_node_tep": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/strategies/multi_node_tep.json", "multi_node_dep": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/strategies/multi_node_dep.json", "kv_store_centralized_mooncake": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/h200.json", "h100": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/h100.json", "b200": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/b200.json", "gb200": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/gb200.json", "b300": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/b300.json", "gb300": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/gb300.json", "mi300x": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/mi300x.json", "mi325x": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/mi325x.json", "mi350x": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/mi350x.json", "mi355x": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/mi355x.json", "arc_pro_b60": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/arc_pro_b60.json", "arc_pro_b70": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/hw/arc_pro_b70.json" } }, "model": { "model_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "min_vllm_version": "0.17.0", "architecture": "moe", "parameter_count": "30B", "active_parameters": "3B", "context_length": 262144, "base_args": [ "--trust-remote-code", "--async-scheduling" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Qwen3 Coder tool-call parser with automatic tool choice", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder" ] }, "reasoning": { "description": "Custom Nano v3 reasoning parser (download plugin: nano_v3_reasoning_parser.py)", "args": [ "--reasoning-parser-plugin", "nano_v3_reasoning_parser.py", "--reasoning-parser", "nano_v3" ] }, "long_context": { "description": "Raise the context window from the default 256K to the model's full 1M tokens. Needs a large KV-cache budget — lower --max-model-len if you are memory-constrained.", "args": [ "--max-model-len", "1048576" ], "env": { "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1" } } }, "opt_in_features": [ "long_context" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 72, "description": "BF16 weights", "extra_args": [ "--kv-cache-dtype", "auto" ] }, "fp8": { "model_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", "precision": "fp8", "vram_minimum_gb": 35, "description": "FP8 weights + FP8 KV cache", "extra_args": [ "--kv-cache-dtype", "fp8" ], "hardware_overrides": { "nvidia": { "extra_args": [ "--moe-backend", "flashinfer_cutlass" ] } }, "json": "/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.json" } }, "kv_offload_support": { "offloading_cpu": "verified", "offloading_fs": "verified" }, "hardware_overrides": { "xpu": { "extra_args": [ "--enforce-eager" ] } }, "guide": "## Overview\n\nNVIDIA Nemotron-3-Nano-30B-A3B is a hybrid-Mamba MoE model (30B total, ~3B active) with\nFP8 and BF16 variants. It supports DGX Spark and Jetson Thor in addition to standard\nHopper/Blackwell servers.\n\n## Prerequisites\n\n- Hardware: 1x H100/H200 or comparable; DGX Spark, Jetson Thor, and Intel Arc Pro B60/B70 supported\n- vLLM >= 0.17.0 (0.28.0 recommended for full support)\n- Docker with NVIDIA Container Toolkit (recommended)\n\n### Pull Docker Image\n\n```bash\ndocker pull --platform linux/amd64 vllm/vllm-openai:v0.28.0\ndocker tag vllm/vllm-openai:v0.28.0 vllm/vllm-openai:deploy\n```\n\nDGX Spark users can build from source (see README) or use the NGC image:\n\n```bash\ndocker pull nvcr.io/nvidia/vllm:25.12.post1-py3\n```\n\nJetson Thor:\n\n```bash\ndocker pull ghcr.io/nvidia-ai-iot/vllm:latest-jetson-thor\n```\n\nIntel XPU (B60 / B70):\n\n```bash\ndocker pull vllm/vllm-openai-xpu:latest\n```\n\n## Launch commands\n\nFP8 with FlashInfer MoE backend (Blackwell/Hopper):\n\n```bash\nvllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 \\\n --trust-remote-code \\\n --async-scheduling \\\n --kv-cache-dtype fp8 \\\n --tensor-parallel-size 1 \\\n --moe-backend flashinfer_cutlass\n```\n\nBF16 (with reasoning + tool parsers — typical for Spark/Thor):\n\n```bash\nwget https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16/resolve/main/nano_v3_reasoning_parser.py\n\nvllm serve nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \\\n --max-num-seqs 8 \\\n --tensor-parallel-size 1 \\\n --max-model-len 262144 \\\n --trust-remote-code \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser-plugin nano_v3_reasoning_parser.py \\\n --reasoning-parser nano_v3\n```\n\n### Intel XPU (B60, B70)\n\nValidated on 4× Intel Arc Pro B60 / B70 (B60 24 GB, B70 32 GB per card) with the\nofficial vLLM XPU image `vllm/vllm-openai-xpu:latest`.\n\nFP8:\n```bash\ndocker run --device /dev/dri \\\n -v /dev/dri/by-path:/dev/dri/by-path --shm-size=16g \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-xpu:latest \\\n nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 \\\n --trust-remote-code \\\n --async-scheduling \\\n --kv-cache-dtype fp8 \\\n --tensor-parallel-size 4 \\\n --enforce-eager \\\n --max-model-len 8192\n```\n\nBF16:\n```bash\ndocker run --device /dev/dri \\\n -v /dev/dri/by-path:/dev/dri/by-path --shm-size=16g \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-xpu:latest \\\n nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 \\\n --trust-remote-code \\\n --async-scheduling \\\n --kv-cache-dtype auto \\\n --tensor-parallel-size 4 \\\n --enforce-eager \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser-plugin nano_v3_reasoning_parser.py \\\n --reasoning-parser nano_v3 \\\n --max-model-len 8192\n```\n\nKey flags:\n- `kv-cache-dtype fp8` for FP8 variant, `auto` for BF16\n- `async-scheduling` reduces host overhead between decode steps\n- `mamba-ssm-cache-dtype float32` for best accuracy, `float16` for speed\n- `max-num-seqs` cap to match client concurrency for lower per-user latency\n\n## Benchmarking\n\n```bash\nvllm bench serve \\\n --model nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 \\\n --trust-remote-code \\\n --dataset-name random \\\n --random-input-len 1024 --random-output-len 1024 \\\n --num-warmups 20 \\\n --ignore-eos \\\n --max-concurrency 1024 \\\n --num-prompts 2048\n```\n\n## Troubleshooting\n\n- Use `--kv-cache-dtype fp8` only with the FP8 checkpoint.\n- Balance TP and `--max-num-seqs` for throughput vs. per-user latency.\n\n## References\n\n- [Nemotron-3-Nano-30B-A3B-BF16](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16)\n- [Nemotron-3-Nano-30B-A3B-FP8](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml", "overlays": [] } }