{ "hf_id": "meta-models/Muse-Glimmer-30B", "meta": { "title": "Muse Glimmer 30B", "slug": "muse-glimmer-30b", "provider": "Muse (Meta)", "description": "Dense 29.6B vision-language model with a ViT-G/14 perception encoder and 128K context, distilled from Muse Spark for local agentic use. Emits channel-scoped reasoning and XML-style ATEM tool calls rather than JSON, so it needs the dedicated `muse_glimmer` tool-call and reasoning parsers.", "date_added": "2026-08-09", "date_updated": "2026-09-09", "tasks": [ "text", "multimodal" ], "performance_headline": "", "related_recipes": [], "default_hardware": "dgx_spark_gb10", "hardware": { "gb300": "verified", "dgx_spark_gb10": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified", "arc_pro_b60": "verified", "arc_pro_b70": "verified", "rtx_5090": "verified", "rtx_5090_2x": "verified" } }, "recommended_command": { "hardware": "dgx_spark_gb10", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:v0.28.0", "command": "vllm serve meta-models/Muse-Glimmer-30B \\\n --generation-config auto \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer", "argv": [ "vllm", "serve", "meta-models/Muse-Glimmer-30B", "--generation-config", "auto", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "muse_glimmer", "--reasoning-parser", "muse_glimmer" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:v0.28.0 meta-models/Muse-Glimmer-30B \\\n --generation-config auto \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:v0.28.0", "meta-models/Muse-Glimmer-30B", "--generation-config", "auto", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "muse_glimmer", "--reasoning-parser", "muse_glimmer" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "blackwell", "compute_arch": "sm121", "display_name": "DGX Spark (GB10)", "description": "NVIDIA DGX Spark · 1× GB10 Grace Blackwell Superchip · 128 GB unified memory", "gpu_count": 1, "vram_gb": 128, "multi_node": false, "restricted": true }, "alternatives": { "kv_store_centralized_mooncake": "/meta-models/Muse-Glimmer-30B/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/meta-models/Muse-Glimmer-30B/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "dgx_spark_gb10": "/meta-models/Muse-Glimmer-30B/hw/dgx_spark_gb10.json", "h100": "/meta-models/Muse-Glimmer-30B/hw/h100.json", "h200": "/meta-models/Muse-Glimmer-30B/hw/h200.json", "b200": "/meta-models/Muse-Glimmer-30B/hw/b200.json", "gb200": "/meta-models/Muse-Glimmer-30B/hw/gb200.json", "b300": "/meta-models/Muse-Glimmer-30B/hw/b300.json", "gb300": "/meta-models/Muse-Glimmer-30B/hw/gb300.json", "rtx_5090": "/meta-models/Muse-Glimmer-30B/hw/rtx_5090.json", "rtx_5090_2x": "/meta-models/Muse-Glimmer-30B/hw/rtx_5090_2x.json", "mi300x": "/meta-models/Muse-Glimmer-30B/hw/mi300x.json", "mi325x": "/meta-models/Muse-Glimmer-30B/hw/mi325x.json", "mi350x": "/meta-models/Muse-Glimmer-30B/hw/mi350x.json", "mi355x": "/meta-models/Muse-Glimmer-30B/hw/mi355x.json", "arc_pro_b60": "/meta-models/Muse-Glimmer-30B/hw/arc_pro_b60.json", "arc_pro_b70": "/meta-models/Muse-Glimmer-30B/hw/arc_pro_b70.json" } }, "model": { "model_id": "meta-models/Muse-Glimmer-30B", "min_vllm_version": "0.28.0", "docker_image": "vllm/vllm-openai:v0.28.0", "architecture": "dense", "parameter_count": "29.6B", "active_parameters": "29.6B", "context_length": 131072, "base_args": [ "--generation-config", "auto" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:v0.28.0", "note": "Recommended: the pinned image ships all dependencies" } } }, "features": { "tool_calling": { "description": "ATEM (XML-style) tool calls with automatic tool choice. Calls arrive in `message.tool_calls`; do not enable guided JSON — this model does not emit JSON tool calls. It emits one tool call per message: several calls arrive as consecutive assistant messages, not a single multi-element `tool_calls` array.", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "muse_glimmer" ] }, "reasoning": { "description": "Channel-scoped chain-of-thought. Surfaces as `message.reasoning` (non-streaming) and `delta.reasoning` (streaming). Effort is controlled by a `Reasoning strength: ` line in the system prompt.", "args": [ "--reasoning-parser", "muse_glimmer" ] }, "spec_decoding": { "hardware": { "rtx_5090": "unsupported" }, "description": "DFlash block-diffusion draft head — predicts a whole block in one forward, verified in parallel by the target. `num_speculative_tokens: 15` is not a tuning knob: the head has `block_size: 16` and slot 0 re-presents the last accepted token, so 15 is what remains.", "args": [ "--max-num-seqs", "32", "--speculative-config", "{\"method\": \"dflash\", \"model\": \"meta-models/Muse-Glimmer-30B-assistant\", \"num_speculative_tokens\": 15}" ] } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 72, "description": "BF16 weights, the reference precision.", "hardware_overrides": { "mi300x": { "extra_args": [ "--attention-backend", "ROCM_AITER_FA" ] }, "mi325x": { "extra_args": [ "--attention-backend", "ROCM_AITER_FA" ] }, "mi355x": { "extra_args": [ "--attention-backend", "ROCM_AITER_FA" ] } } }, "fp8_block": { "precision": "fp8", "model_id": "RedHatAI/Muse-Glimmer-30B-FP8-block", "vram_minimum_gb": 40, "description": "FP8_BLOCK (E4M3) weights with 128x128 block scaling, dynamic activations. 33 GB on disk, fits one GPU.", "json": "/RedHatAI/Muse-Glimmer-30B-FP8-block.json" }, "nvfp4": { "precision": "nvfp4", "model_id": "Inferact/Muse-Glimmer-30B-NVFP4-W4A4", "vram_minimum_gb": 31, "description": "ModelOpt NVFP4 W4A4 — weights and activations at 4 bits, group size 16. 25.42 GB on disk, fits one GPU.", "json": "/Inferact/Muse-Glimmer-30B-NVFP4-W4A4.json" } }, "hardware_overrides": { "amd": { "extra_args": [], "extra_env": { "VLLM_ROCM_USE_AITER": "1" } }, "xpu": { "extra_args": [ "--enforce-eager" ] } }, "guide": "## Overview\n\nMuse Glimmer 30B is a dense vision-language model built for agentic work on\nconsumer hardware: a 52-layer text decoder (hidden 6656) plus a ~1.8B ViT-G/14\nperception encoder, 128K trained context, BF16. Apache 2.0, knowledge cutoff\nJanuary 4 2026, trained on 100+ languages.\n\nWhat makes it unlike most recipes here is its output format. It does not emit\nJSON tool calls and it does not wrap reasoning in `` tags. Every turn is\nwritten as a sequence of channel-scoped messages:\n\n```\nto=self<|message|>...chain of thought...<|eom|>\n<|start|>assistant to=<|message|>\n\">\n\">value\n\n<|eom|>\n<|start|>assistant to=user<|message|>...final answer...<|eot|>\n```\n\nBoth `--tool-call-parser muse_glimmer` and `--reasoning-parser muse_glimmer` key off that\nframing, so run them together. The reasoning parser also forces\n`skip_special_tokens=False` — without it the markers are stripped before\nparsing and both channels collapse into content.\n\n## Prerequisites\n\n- **Hardware**: DGX Spark. FP4 weights might also run on 5090.\n- **Image**: `vllm/vllm-openai:v0.28.0`.\n\n## Checkpoints\n\n| variant | repo | size | notes |\n|---|---|---|---|\n| BF16 | [`meta-models/Muse-Glimmer-30B`](https://huggingface.co/meta-models/Muse-Glimmer-30B) | 59.58 GB | reference precision |\n| FP8 | [`RedHatAI/Muse-Glimmer-30B-FP8-block`](https://huggingface.co/RedHatAI/Muse-Glimmer-30B-FP8-block) | 32.78 GB | FP8 block-scaled weights, dynamic activations (vision tower + embeddings BF16) |\n| NVFP4 | [`Inferact/Muse-Glimmer-30B-NVFP4-W4A4`](https://huggingface.co/Inferact/Muse-Glimmer-30B-NVFP4-W4A4) | 25.42 GB | ModelOpt NVFP4, **W4A4**, group size 16 |\n| draft | [`meta-models/Muse-Glimmer-30B-assistant`](https://huggingface.co/meta-models/Muse-Glimmer-30B-assistant) | 5.11 GB | DFlash draft head for speculative decoding |\n\nThe NVFP4 build quantizes **both weights and activations** to 4 bits across 364\nlanguage-model projections (52 layers x 7); embeddings, `lm_head` and the vision\ntower stay unquantized, which is why it is 25 GB rather than the ~15 GB a\nuniform 4-bit quant would give. vLLM detects it from `config.json`\n(`quant_method: modelopt`, `quant_algo: NVFP4`) and selects `modelopt_fp4` with\nfused activation quantization — no extra flags, the launch command below is\nunchanged apart from the model path. NVFP4 kernels are Blackwell-only.\n\n## Launching the Server\n\n```bash\nvllm serve /model \\\n --served-model-name muse-glimmer \\\n --tensor-parallel-size 4 \\\n --gpu-memory-utilization 0.92 \\\n --max-model-len 131072 \\\n --max-num-seqs 64 \\\n --enable-auto-tool-choice --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer \\\n --generation-config auto\n```\n\nAt TP=4 on GB300 this reports a 26.8M-token KV pool and ~204x max concurrency\nat the full 128K context.\n\n## Running on DGX Spark (GB10)\n\nMuse Glimmer was built for local deployment and runs on a DGX Spark as an\nordinary single-card deployment — **no special configuration**. GB10 is aarch64\nlike GB300 so the same image applies, and the launch command above is unchanged\napart from having one GPU to run on:\n\n```bash\nvllm serve /model \\\n --served-model-name muse-glimmer \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer \\\n --generation-config auto\n```\n\nThere is no tensor parallelism to configure: GB10 is a single GPU, so TP is 1.\nBF16 weights are ~55 GiB against 128 GB of unified memory, leaving the rest for\nthe KV cache and the perception encoder. The **NVFP4** checkpoint is 25.42 GB,\nwhich leaves considerably more of that shared pool for KV and the host — the\nbetter fit of the two on this hardware.\n\n## Running on MI300X/MI325X/MI355X\n\nUse the official `vllm/vllm-openai-rocm:nightly` image.\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nvllm serve /model \\\n --served-model-name muse-glimmer \\\n --tensor-parallel-size 4 \\\n --max-model-len 131072 \\\n --enable-auto-tool-choice \\\n --tool-call-parser muse_glimmer \\\n --reasoning-parser muse_glimmer \\\n --generation-config auto \\\n --attention-backend ROCM_AITER_FA\n```\n\nAt TP=1 on MI300X this reports an 6.66M-token KV pool and ~50.82x max concurrency\nat the full 128K context.\n\nAt TP=4 on MI300X this reports an 17.79M-token KV pool and ~135.73x max concurrency\nat the full 128K context.\n\nAt TP=1 on MI355X this reports an 11.64M-token KV pool and ~88.86x max concurrency\nat the full 128K context.\n\nAt TP=4 on MI355X this reports an 27.68M-token KV pool and ~211.22x max concurrency\nat the full 128K context.\n\n## Running on Intel Arc Pro (XPU) and Xeon (CPU)\n\nvalidated on Intel Arc Pro B70, Arc Pro B60, and Xeon 6.\n\n**Intel GPU (XPU):**\n\n```bash\ndocker run -it --rm --network=host --ipc=host --privileged \\\n --device /dev/dri:/dev/dri \\\n -v /dev/dri/by-path:/dev/dri/by-path \\\n --entrypoint bash \\\n vllm/vllm-openai-xpu:latest\n```\n\nServe inside the container — `--enforce-eager` is required on the XPU backend:\n\n```bash\nvllm serve meta-models/Muse-Glimmer-30B \\\n --tensor-parallel-size 4 \\\n --reasoning-parser muse_glimmer \\\n --enforce-eager\n```\n\n## Sampling\n\nUse the published settings:\n\n```\ntemperature = 1.0\ntop_p = 0.95\ntop_k = 64\n```\n\nDo not run it greedy — it is a reasoning model, and greedy is not reproducible\nhere either: identical requests at `temperature: 0` with a fixed `seed` returned\n70, 80, and 86 completion tokens across runs.\n\n**Reasoning strength.** Effort is set with a `Reasoning strength: ` line\nin the system prompt, one of `low` / `medium` / `high` / `xhigh`. Use `high` or\n`xhigh` for coding and agentic tasks.\n\n## RTX 5090 (32 GB, consumer Blackwell)\n\n The `nvfp4` variant runs on a **single** RTX 5090 with no overrides: 131,072\n context, a 179,647-token KV cache (1.37x concurrency at full length), 28.8 GB\n of 32.6 GB used, and 68.8 tok/s on a 600-token greedy generation. Tool calling\n and vision both work. `vram_minimum_gb: 31` is accurate — the margin is real\n but thin.\n\n **Speculative decoding needs a second card.** The draft head is 5.11 GB and the\n nvfp4 weights are 25.42 GB, so on one 32 GB card vLLM OOMs during init with\n roughly 400 MiB free. On 2x 5090 with `--tensor-parallel-size 2` it is well\n worth the second card — decode roughly triples.\n\n ### 2x RTX 5090\n\n Verified with the drafter enabled, and worth the second card: decode goes from\n 68.8 tok/s to ~240 tok/s.\n\n `--gpu-memory-utilization` only sizes the KV cache; it does not gate\n initialisation. Measured across the range on the base recipe (131,072 ctx,\n bf16 KV) plus DFlash:\n\n | `--gpu-memory-utilization` | KV cache | concurrency @131K |\n |---|---|---|\n | 0.92 | 1,202,076 | 9.17x |\n | 0.88 | 1,054,407 | 8.04x |\n | 0.84 | 906,720 | 6.92x |\n | 0.80 | 759,051 | 5.79x |\n\n All seven steps from 0.92 down to 0.80 initialised cleanly, including the\n default 0.92. Decode was 212-261 tok/s with no trend attributable to\n utilisation — the spread is run-to-run noise (another model shared the host\n during the early steps) — so treat ~240 tok/s as the figure and pick\n utilisation purely for the cache size you want.\n\n If you *extend* beyond the recipe — a longer `--max-model-len` via rope\n scaling, or `--kv-cache-dtype fp8` — the arithmetic changes and 0.92 can OOM:\n at 262,144 with fp8 KV, vLLM sized a 2,594,229-token cache and then failed to\n allocate 396 MiB for the speculator. Lowering to 0.80 fixed it. That is a\n consequence of the extension, not of the recipe as written.\n\n Not tested at 2x: vision, and concurrent load at `--max-num-seqs 64`.\n\n ### fp8_block on sm120 needs DeepGEMM off\n\n The `fp8_block` variant does not load on RTX 5090 with DeepGEMM enabled. It\n fails in `process_weights_after_loading`, before the engine starts:\n\n ```\n RuntimeError: Assertion error (deepgemm/csrc/apis/layout.hpp:60): Unknown SF transformation\n ```\n\n vLLM treats capability family 120 as a supported DeepGEMM target, so the path\n is selected and then DeepGEMM rejects the scale-factor layout. Filed as\n vllm-project/vllm#51884. Set both to work around it:\n\n ```bash\n VLLM_USE_DEEP_GEMM=0 VLLM_MOE_USE_DEEP_GEMM=0\n ```\n\n With that, fp8_block serves fine on 2x 5090 and is the better choice than\n nvfp4 if you have the second card — 8-bit weights instead of W4A4, and faster:\n 177.6 tok/s with the drafter at 262,144 context (rope-extended, so outside the\n recipe as written). nvfp4 is unaffected and loads with DeepGEMM at its default.\n\n A note on `max_tokens`: the channel-scoped output means a tight budget can\n truncate before the final channel closes, returning empty `content` with\n `finish_reason: stop`. Give it room.\n\n## Speculative decoding\n\nMuse Glimmer ships a DFlash block-diffusion draft head. Rather than proposing\none token at a time, it predicts a whole block in a single forward — the slots\nattend to each other bidirectionally — and the target verifies them in parallel.\n\nDFlash uses 16 query positions per active request during warmup. To keep the\nexpanded batch within vLLM's token budget, replace the earlier sequence limit with:\n\n```bash\n--max-num-seqs 32 \\\n --speculative-config '{\"method\": \"dflash\", \"model\": \"meta-models/Muse-Glimmer-30B-assistant\", \"num_speculative_tokens\": 15}'\n```\n\n**`num_speculative_tokens: 15` is fixed, not tuned.** The head has\n`block_size: 16` and slot 0 re-presents the last accepted token, so 15 slots\nremain to predict.\n\nThe head is 5 layers / 5.11 GB, reads the target's residual stream at layers\n`[1, 13, 25, 37, 49]`, and ties its embedding and output projection to the\ntarget — so it stores neither and is useless on its own. It must be paired with\nthe specific target it was distilled against, not merely a compatible one.\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/meta-models/Muse-Glimmer-30B.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/meta-models/Muse-Glimmer-30B.yaml", "overlays": [] } }