{
  "hf_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16",
  "meta": {
    "title": "Nemotron-3-Nano-Omni-30B-A3B-Reasoning",
    "slug": "nemotron-3-nano-omni-30b-a3b-reasoning",
    "provider": "NVIDIA",
    "description": "Mamba2-Transformer hybrid MoE omnimodal model (31B total / 3B active) with unified video, audio, image, and text understanding; reasoning + tool calling; BF16, FP8, and NVFP4 variants",
    "date_added": "2026-04-28",
    "date_updated": "2026-04-29",
    "tasks": [
      "multimodal",
      "text"
    ],
    "related_recipes": [
      "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
      "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16"
    ],
    "hardware": {
      "h100": "verified",
      "h200": "verified",
      "b200": "verified"
    }
  },
  "recommended_command": {
    "hardware": "h200",
    "strategy": "single_node_tp",
    "variant": "default",
    "node_count": 1,
    "deploy_type": "single_node",
    "env": {},
    "docker_image": "vllm/vllm-openai:latest",
    "command": "vllm serve nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 \\\n  --trust-remote-code \\\n  --max-model-len 131072 \\\n  --media-io-kwargs '{\"video\": {\"num_frames\": 512, \"fps\": 1}}' \\\n  --video-pruning-rate 0.5 \\\n  --tensor-parallel-size 1 \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder \\\n  --reasoning-parser nemotron_v3",
    "argv": [
      "vllm",
      "serve",
      "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16",
      "--trust-remote-code",
      "--max-model-len",
      "131072",
      "--media-io-kwargs",
      "{\"video\": {\"num_frames\": 512, \"fps\": 1}}",
      "--video-pruning-rate",
      "0.5",
      "--tensor-parallel-size",
      "1",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "qwen3_coder",
      "--reasoning-parser",
      "nemotron_v3"
    ],
    "docker_command": "docker run --gpus all \\\n  --privileged --ipc=host -p 8000:8000 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  vllm/vllm-openai:latest nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 \\\n  --trust-remote-code \\\n  --max-model-len 131072 \\\n  --media-io-kwargs '{\"video\": {\"num_frames\": 512, \"fps\": 1}}' \\\n  --video-pruning-rate 0.5 \\\n  --tensor-parallel-size 1 \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder \\\n  --reasoning-parser nemotron_v3",
    "docker_argv": [
      "docker",
      "run",
      "--gpus",
      "all",
      "--privileged",
      "--ipc=host",
      "-p",
      "8000:8000",
      "-v",
      "~/.cache/huggingface:/root/.cache/huggingface",
      "vllm/vllm-openai:latest",
      "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16",
      "--trust-remote-code",
      "--max-model-len",
      "131072",
      "--media-io-kwargs",
      "{\"video\": {\"num_frames\": 512, \"fps\": 1}}",
      "--video-pruning-rate",
      "0.5",
      "--tensor-parallel-size",
      "1",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "qwen3_coder",
      "--reasoning-parser",
      "nemotron_v3"
    ],
    "strategy_spec": {
      "name": "single_node_tp",
      "deploy_type": "single_node",
      "display_name": "Tensor Parallel",
      "short_label": "TP",
      "orientation": "latency",
      "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n",
      "hardware_match": {
        "min_gpus": 1,
        "max_gpus": 8,
        "multi_node": false
      },
      "vllm_args": [],
      "parallel_flag": "--tensor-parallel-size"
    },
    "hardware_profile": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H200",
      "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node",
      "gpu_count": 8,
      "vram_gb": 1128,
      "multi_node": false
    },
    "alternatives": {
      "multi_node_tp": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/strategies/multi_node_tp.json",
      "kv_store_centralized_mooncake": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/strategies/kv_store_centralized_mooncake.json",
      "kv_store_distributed_mooncake": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/strategies/kv_store_distributed_mooncake.json"
    },
    "by_hardware": {
      "h200": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/h200.json",
      "h100": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/h100.json",
      "b200": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/b200.json",
      "gb200": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/gb200.json",
      "b300": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/b300.json",
      "gb300": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/gb300.json",
      "mi300x": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/mi300x.json",
      "mi325x": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/mi325x.json",
      "mi350x": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/mi350x.json",
      "mi355x": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16/hw/mi355x.json"
    }
  },
  "model": {
    "model_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16",
    "min_vllm_version": "0.20.0",
    "architecture": "moe",
    "parameter_count": "31B",
    "active_parameters": "3B",
    "context_length": 262144,
    "base_args": [
      "--trust-remote-code",
      "--max-model-len",
      "131072",
      "--media-io-kwargs",
      "{\"video\": {\"num_frames\": 512, \"fps\": 1}}",
      "--video-pruning-rate",
      "0.5"
    ],
    "base_env": {},
    "install": {
      "pip": {
        "command": "uv pip install \"vllm[audio]==0.28.0\"",
        "note": "Pinned to 0.28.0 with the audio extra (required for audio + use_audio_in_video)."
      },
      "docker": {
        "command": "docker pull vllm/vllm-openai:latest"
      }
    }
  },
  "features": {
    "tool_calling": {
      "description": "Qwen3 Coder tool-call parser with automatic tool choice",
      "args": [
        "--enable-auto-tool-choice",
        "--tool-call-parser",
        "qwen3_coder"
      ]
    },
    "reasoning": {
      "description": "Nemotron v3 reasoning parser (chain-of-thought with <think> tags)",
      "args": [
        "--reasoning-parser",
        "nemotron_v3"
      ]
    }
  },
  "opt_in_features": [],
  "variants": {
    "default": {
      "precision": "bf16",
      "vram_minimum_gb": 75,
      "description": "BF16 weights — full-precision reference"
    },
    "fp8": {
      "model_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8",
      "precision": "fp8",
      "vram_minimum_gb": 38,
      "description": "ModelOpt FP8 weights + FP8 KV cache",
      "extra_args": [
        "--kv-cache-dtype",
        "fp8"
      ],
      "json": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8.json"
    },
    "nvfp4": {
      "model_id": "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4",
      "precision": "nvfp4",
      "vram_minimum_gb": 28,
      "description": "ModelOpt NVFP4 weights — Blackwell-only; smallest footprint",
      "extra_args": [
        "--kv-cache-dtype",
        "fp8"
      ],
      "json": "/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4.json"
    }
  },
  "kv_offload_support": {
    "offloading_cpu": "verified",
    "offloading_fs": "verified"
  },
  "hardware_overrides": {},
  "guide": "## Overview\n\nNVIDIA Nemotron-3-Nano-Omni-30B-A3B-Reasoning is a Mamba2-Transformer hybrid\nMoE omnimodal model (31B total / 3B active) that unifies video, audio, image,\nand text understanding. It is built on the Nemotron-3-Nano-30B-A3B LLM\nbackbone with a CRADIO v4-H vision encoder and a Parakeet speech encoder, and\nships in BF16, ModelOpt FP8, and ModelOpt NVFP4 variants.\n\nCapabilities:\n- Video (mp4, up to 2 minutes, sampled at 1–2 FPS / 128–256 frames)\n- Audio (wav, mp3, up to 1 hour, ≥8 kHz)\n- Image (jpeg, png)\n- Text (English, up to 256K context)\n- Reasoning with chain-of-thought (`<think>` tags)\n- Tool calling\n- Word-level timestamps for transcription\n\n## Prerequisites\n\n- vLLM **0.28.0** (pinned: `pip install vllm[audio]==0.28.0`,\n  or pull `vllm/vllm-openai:v0.28.0`)\n- Hardware: 1× B200 / H200 / H100 (single-GPU TP1 is the documented profile)\n- The `audio` extra is required for any audio input, including\n  `use_audio_in_video=true`\n\n### Pull the Docker image\n\n```bash\n# CUDA 13:\ndocker pull vllm/vllm-openai:v0.28.0\n# CUDA 12.9:\ndocker pull vllm/vllm-openai:v0.28.0-cu129\n```\n\n## Launch command\n\nGeneral single-GPU invocation (B200 / H200 / H100):\n\n```bash\nvllm serve nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16 \\\n  --served-model-name nemotron \\\n  --host 0.0.0.0 \\\n  --port 5000 \\\n  --tensor-parallel-size 1 \\\n  --max-model-len 131072 \\\n  --trust-remote-code \\\n  --video-pruning-rate 0.5 \\\n  --media-io-kwargs '{\"video\": {\"num_frames\": 512, \"fps\": 1}}' \\\n  --reasoning-parser nemotron_v3 \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder\n```\n\nSwap the model id for the FP8 or NVFP4 checkpoint and add `--kv-cache-dtype fp8`:\n\n```bash\nvllm serve nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4 \\\n  --kv-cache-dtype fp8 \\\n  ...\n```\n\n### Platform-specific notes\n\n- **RTX Pro:** append `--moe-backend triton` (FlashInfer + RTX Pro bug).\n- **NVFP4 + TP>1:** append `--moe-backend flashinfer_cutlass` (TRTLLM_GEN MoE\n  kernel bug at TP>1 on NVFP4).\n- **DGX Spark (aarch64):** unified LPDDR5X memory; lower\n  `--gpu-memory-utilization` to 0.70 and reduce `--max-model-len` (e.g. 32768)\n  if you hit OOM. Use `--max-num-seqs 8`.\n\n## Recommended sampling\n\n| Mode | temperature | top_p | top_k | max_tokens | reasoning_budget |\n|------|------------:|------:|------:|-----------:|------------------:|\n| Thinking | 0.6 | 0.95 | — | 20480 | 16384 |\n| Instruct | 0.2 | — | 1 | 1024 | — |\n\nToggle thinking mode via `chat_template_kwargs={\"enable_thinking\": true}`\n(default) or `false` to disable.\n\n## Client usage\n\n```python\nfrom openai import OpenAI\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"\")\nresp = client.chat.completions.create(\n    model=\"nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4\",\n    messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n    extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}},\n)\n```\n\n## Benchmarking\n\n```bash\nvllm bench serve \\\n  --model nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8 \\\n  --trust-remote-code \\\n  --dataset-name random \\\n  --random-input-len 1024 --random-output-len 1024 \\\n  --num-warmups 20 --ignore-eos \\\n  --max-concurrency 1024 --num-prompts 2048\n```\n\n## References\n\n- [BF16 model card](https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16)\n- [FP8 model card](https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-FP8)\n- [NVFP4 model card](https://huggingface.co/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4)\n- [Nemotron 3 Nano LLM backbone](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16)\n",
  "source": {
    "source": "vllm",
    "repo": "vllm-project/vllm",
    "ref": "main",
    "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb",
    "path": "recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16.yaml",
    "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16.yaml",
    "overlays": []
  }
}