{
  "hf_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16",
  "meta": {
    "title": "NVIDIA Nemotron-3-Super-120B-A12B",
    "slug": "nemotron-3-super-120b-a12b",
    "provider": "NVIDIA",
    "description": "NVIDIA Nemotron-3-Super Mamba-hybrid latent-MoE (~120B total / ~12B active) with BF16, FP8, and NVFP4 variants",
    "date_added": "2026-04-28",
    "date_updated": "2026-10-08",
    "tasks": [
      "text"
    ],
    "related_recipes": [
      "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
    ],
    "hardware": {
      "h100": "verified",
      "h200": "verified",
      "b200": "verified",
      "dgx_station_gb300": "verified",
      "dgx_spark_gb10": "verified",
      "rtx_pro_6000": "verified"
    }
  },
  "recommended_command": {
    "hardware": "h200",
    "strategy": "single_node_tp",
    "variant": "default",
    "node_count": 1,
    "deploy_type": "single_node",
    "env": {},
    "docker_image": "vllm/vllm-openai:latest",
    "command": "vllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 \\\n  --trust-remote-code \\\n  --kv-cache-dtype fp8 \\\n  --tensor-parallel-size 8 \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder \\\n  --reasoning-parser nemotron_v3",
    "argv": [
      "vllm",
      "serve",
      "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16",
      "--trust-remote-code",
      "--kv-cache-dtype",
      "fp8",
      "--tensor-parallel-size",
      "8",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "qwen3_coder",
      "--reasoning-parser",
      "nemotron_v3"
    ],
    "docker_command": "docker run --gpus all \\\n  --privileged --ipc=host -p 8000:8000 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  vllm/vllm-openai:latest nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 \\\n  --trust-remote-code \\\n  --kv-cache-dtype fp8 \\\n  --tensor-parallel-size 8 \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder \\\n  --reasoning-parser nemotron_v3",
    "docker_argv": [
      "docker",
      "run",
      "--gpus",
      "all",
      "--privileged",
      "--ipc=host",
      "-p",
      "8000:8000",
      "-v",
      "~/.cache/huggingface:/root/.cache/huggingface",
      "vllm/vllm-openai:latest",
      "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16",
      "--trust-remote-code",
      "--kv-cache-dtype",
      "fp8",
      "--tensor-parallel-size",
      "8",
      "--enable-auto-tool-choice",
      "--tool-call-parser",
      "qwen3_coder",
      "--reasoning-parser",
      "nemotron_v3"
    ],
    "strategy_spec": {
      "name": "single_node_tp",
      "deploy_type": "single_node",
      "display_name": "Tensor Parallel",
      "short_label": "TP",
      "orientation": "latency",
      "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n",
      "hardware_match": {
        "min_gpus": 1,
        "max_gpus": 8,
        "multi_node": false
      },
      "vllm_args": [],
      "parallel_flag": "--tensor-parallel-size"
    },
    "hardware_profile": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H200",
      "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node",
      "gpu_count": 8,
      "vram_gb": 1128,
      "multi_node": false
    },
    "alternatives": {
      "multi_node_tp": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/strategies/multi_node_tp.json",
      "multi_node_tp_pp": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/strategies/multi_node_tp_pp.json",
      "multi_node_tep": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/strategies/multi_node_tep.json",
      "multi_node_dep": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/strategies/multi_node_dep.json",
      "kv_store_centralized_mooncake": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/strategies/kv_store_centralized_mooncake.json",
      "kv_store_distributed_mooncake": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/strategies/kv_store_distributed_mooncake.json"
    },
    "by_hardware": {
      "h200": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/h200.json",
      "h100": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/h100.json",
      "b200": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/b200.json",
      "gb200": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/gb200.json",
      "b300": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/b300.json",
      "gb300": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/gb300.json",
      "dgx_station_gb300": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/dgx_station_gb300.json",
      "dgx_spark_gb10": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/dgx_spark_gb10.json",
      "rtx_pro_6000": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/rtx_pro_6000.json",
      "mi300x": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/mi300x.json",
      "mi325x": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/mi325x.json",
      "mi350x": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/mi350x.json",
      "mi355x": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16/hw/mi355x.json"
    }
  },
  "model": {
    "model_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16",
    "min_vllm_version": "0.17.1",
    "architecture": "moe",
    "parameter_count": "120B",
    "active_parameters": "12B",
    "context_length": 262144,
    "base_args": [
      "--trust-remote-code"
    ],
    "base_env": {},
    "install": {
      "pip": {
        "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto"
      },
      "docker": {
        "command": "docker pull vllm/vllm-openai:latest"
      }
    }
  },
  "features": {
    "tool_calling": {
      "description": "Qwen3 Coder tool-call parser with automatic tool choice",
      "args": [
        "--enable-auto-tool-choice",
        "--tool-call-parser",
        "qwen3_coder"
      ]
    },
    "reasoning": {
      "description": "Built-in nemotron_v3 reasoning parser (vLLM >= 0.17.1)",
      "args": [
        "--reasoning-parser",
        "nemotron_v3"
      ]
    },
    "spec_decoding": {
      "description": "MTP speculative decoding for accelerated inference",
      "args": [
        "--speculative-config",
        "{\"method\":\"mtp\",\"num_speculative_tokens\":3}"
      ]
    },
    "long_context": {
      "description": "Raise the context window from the default 256K to the model's full 1M tokens. Needs a large KV-cache budget — lower --max-model-len if you are memory-constrained.",
      "args": [
        "--max-model-len",
        "1048576"
      ],
      "env": {
        "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1"
      }
    }
  },
  "opt_in_features": [
    "spec_decoding",
    "long_context"
  ],
  "variants": {
    "default": {
      "precision": "bf16",
      "vram_minimum_gb": 298,
      "description": "BF16 weights (FP8 KV cache recommended)",
      "extra_args": [
        "--kv-cache-dtype",
        "fp8"
      ]
    },
    "fp8": {
      "model_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8",
      "precision": "fp8",
      "vram_minimum_gb": 149,
      "description": "FP8 weights + FP8 KV cache",
      "extra_args": [
        "--kv-cache-dtype",
        "fp8"
      ],
      "json": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8.json"
    },
    "nvfp4": {
      "model_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4",
      "precision": "nvfp4",
      "vram_minimum_gb": 75,
      "description": "NVFP4 weights for Blackwell",
      "hardware_overrides": {
        "rtx_pro_6000": {
          "extra_args": [
            "--max-num-seqs",
            "8"
          ]
        },
        "dgx_station_gb300": {
          "extra_args": [
            "--max-model-len",
            "262144",
            "--gpu-memory-utilization",
            "0.92",
            "--load-format",
            "fastsafetensors",
            "--attention-backend",
            "flashinfer",
            "--max-num-batched-tokens",
            "16384",
            "--kv-cache-dtype",
            "fp8",
            "--mamba-cache-mode",
            "align",
            "--mamba-ssm-cache-dtype",
            "float32",
            "--enable-prefix-caching"
          ],
          "extra_env": {
            "VLLM_FLOAT32_MATMUL_PRECISION": "high",
            "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1"
          }
        }
      },
      "json": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4.json"
    },
    "base_bf16": {
      "model_id": "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16",
      "precision": "bf16",
      "vram_minimum_gb": 298,
      "description": "Pre-RL base checkpoint (BF16)",
      "json": "/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16.json"
    }
  },
  "kv_offload_support": {
    "offloading_cpu": "verified",
    "offloading_fs": "verified"
  },
  "hardware_overrides": {},
  "guide": "## Overview\n\nNVIDIA Nemotron-3-Super-120B-A12B is a hybrid-Mamba latent-MoE model (~120B total,\n~12B active per token) trained for general reasoning, tool use, and agentic workflows.\nIt supports a 1M-token context window and Multi-Token Prediction (MTP). Variants ship\nin BF16, FP8, and NVFP4 (Blackwell). A pre-RL Base BF16 checkpoint is also available\nfor downstream fine-tuning.\n\n## Prerequisites\n\n- Hardware: 4-8x H100/H200/B200/RTX Pro 6000, DGX Station, or DGX Spark\n- vLLM >= 0.17.1\n- Docker with NVIDIA Container Toolkit (recommended)\n\n## Launch commands\n\nReference command from the [vLLM blog](https://vllm.ai/blog/nemotron-3-super) (BF16, 4x H100, FP8 KV cache):\n\n```bash\nvllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16 \\\n  --kv-cache-dtype fp8 \\\n  --tensor-parallel-size 4 \\\n  --trust-remote-code \\\n  --served-model-name nemotron \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder \\\n  --reasoning-parser nemotron_v3\n```\n\nFP8 weights:\n\n```bash\nvllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 \\\n  --kv-cache-dtype fp8 \\\n  --tensor-parallel-size 4 \\\n  --trust-remote-code \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder \\\n  --reasoning-parser nemotron_v3\n```\n\nNVFP4 (Blackwell only):\n\n```bash\nvllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 \\\n  --tensor-parallel-size 2 \\\n  --trust-remote-code \\\n  --enable-auto-tool-choice \\\n  --tool-call-parser qwen3_coder \\\n  --reasoning-parser nemotron_v3\n```\n\n### DGX Spark (GB10)\n\nThe NVFP4 variant runs on a single DGX Spark (GB10) at TP=1 inside the\n`vllm/vllm-openai:v0.28.0-ubuntu2404` container. The command below is the optimal\nconfig tuned end-to-end on DGX Spark: the full 262144-token context fits with an FP8\nKV cache and a float32 Mamba SSM cache (`VLLM_ALLOW_LONG_MAX_MODEL_LEN=1` allows the\nlong context, `VLLM_FLOAT32_MATMUL_PRECISION=high` keeps the Mamba scan numerically\nstable), `--gpu-memory-utilization 0.8` leaves headroom in the 128 GB unified memory,\nbatching is sized for a single GB10 (`--max-num-seqs 8`,\n`--max-num-batched-tokens 16384`), `--load-format fastsafetensors` speeds up weight\nloading, and MTP speculative decoding plus prefix caching are enabled for lower\nlatency:\n\n```bash\n# Use container: vllm/vllm-openai:v0.28.0-ubuntu2404\n\nVLLM_FLOAT32_MATMUL_PRECISION=\"high\" \\\nVLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \\\nvllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 \\\n    --host 0.0.0.0 \\\n    --port 8000 \\\n    --max-model-len 262144 \\\n    --gpu-memory-utilization 0.8 \\\n    --max-num-seqs 8 \\\n    --load-format fastsafetensors \\\n    --attention-backend flashinfer \\\n    --max-num-batched-tokens 16384 \\\n    --kv-cache-dtype fp8 \\\n    --mamba-cache-mode align \\\n    --mamba-ssm-cache-dtype float32 \\\n    --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' \\\n    --enable-prefix-caching \\\n    --reasoning-parser nemotron_v3 \\\n    --enable-auto-tool-choice \\\n    --tool-call-parser qwen3_coder\n```\n\n### RTX Pro 6000 NVFP4\n\nThe NVFP4 variant runs on a single RTX Pro 6000 at TP=1. Requires vLLM v0.24.0 or later. A few notes:\n\n- **Set `--gpu-memory-utilization 0.92`** to avoid an OOM at startup. If you encounter an OOM, try adjusting the value.\n- **Set `VLLM_FLOAT32_MATMUL_PRECISION=\"high\"`** to use high precision for the matmul operations.\n- **Set `VLLM_ALLOW_LONG_MAX_MODEL_LEN=1`** to allow the model to use the full context length.\n\n```bash\n# Use container: vllm/vllm-openai:v0.28.0-ubuntu2404\n\nVLLM_FLOAT32_MATMUL_PRECISION=\"high\" \\\nVLLM_ALLOW_LONG_MAX_MODEL_LEN=1 \\\nvllm serve nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 \\\n    --max-model-len 262144 \\\n    --gpu-memory-utilization 0.92 \\\n    --max-num-seqs 8 \\\n    --max-num-batched-tokens 16384 \\\n    --kv-cache-dtype fp8 \\\n    --mamba-cache-mode align \\\n    --mamba-ssm-cache-dtype float32 \\\n    --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}' \\\n    --enable-prefix-caching \\\n    --reasoning-parser nemotron_v3 \\\n    --enable-auto-tool-choice \\\n    --tool-call-parser qwen3_coder\n```\n\n## Benchmarking\n\n```bash\nvllm bench serve \\\n  --model nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 \\\n  --trust-remote-code \\\n  --dataset-name random \\\n  --random-input-len 1024 --random-output-len 1024 \\\n  --num-warmups 20 \\\n  --ignore-eos \\\n  --max-concurrency 1024 \\\n  --num-prompts 2048\n```\n\n## References\n\n- [vLLM blog: Nemotron-3-Super](https://vllm.ai/blog/nemotron-3-super)\n- [vLLM blog: vLLM on DGX Spark](https://vllm.ai/blog/2026-06-01-vllm-dgx-spark)\n- [Nemotron-3-Super-120B-A12B-BF16](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16)\n- [Nemotron-3-Super-120B-A12B-FP8](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8)\n- [Nemotron-3-Super-120B-A12B-NVFP4](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4)\n- [Nemotron-3-Super-120B-A12B-Base-BF16](https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-Base-BF16)\n",
  "source": {
    "source": "vllm",
    "repo": "vllm-project/vllm",
    "ref": "main",
    "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb",
    "path": "recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16.yaml",
    "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16.yaml",
    "overlays": []
  }
}