{ "hf_id": "inclusionAI/Ling-3.0-flash", "meta": { "title": "Ling-3.0-flash", "slug": "ling-3-0-flash", "provider": "inclusionAI", "description": "Ling-3.0-flash MoE model with BF16, FP8, FP4, and INT4 checkpoints, native MTP, and an external DSpark draft model", "date_updated": "2026-09-09", "tasks": [ "text" ], "performance_headline": "BF16, FP8, FP4, and INT4 support NVIDIA H20/H200 and AMD MI300X/MI355X; DGX Spark verified TP1 FP4/INT4", "hardware": { "h200": "verified", "dgx_spark_gb10": "verified", "mi300x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve inclusionAI/Ling-3.0-flash \\\n --trust-remote-code \\\n --dtype bfloat16 \\\n --gpu-memory-utilization 0.9 \\\n --enable-chunked-prefill \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --enable-prefix-caching \\\n --tensor-parallel-size 4 \\\n --enable-auto-tool-choice \\\n --tool-call-parser ling3 \\\n --reasoning-parser ling3", "argv": [ "vllm", "serve", "inclusionAI/Ling-3.0-flash", "--trust-remote-code", "--dtype", "bfloat16", "--gpu-memory-utilization", "0.9", "--enable-chunked-prefill", "--compilation-config", "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--enable-prefix-caching", "--tensor-parallel-size", "4", "--enable-auto-tool-choice", "--tool-call-parser", "ling3", "--reasoning-parser", "ling3" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest inclusionAI/Ling-3.0-flash \\\n --trust-remote-code \\\n --dtype bfloat16 \\\n --gpu-memory-utilization 0.9 \\\n --enable-chunked-prefill \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --enable-prefix-caching \\\n --tensor-parallel-size 4 \\\n --enable-auto-tool-choice \\\n --tool-call-parser ling3 \\\n --reasoning-parser ling3", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "inclusionAI/Ling-3.0-flash", "--trust-remote-code", "--dtype", "bfloat16", "--gpu-memory-utilization", "0.9", "--enable-chunked-prefill", "--compilation-config", "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--enable-prefix-caching", "--tensor-parallel-size", "4", "--enable-auto-tool-choice", "--tool-call-parser", "ling3", "--reasoning-parser", "ling3" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy โ€” works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e ยท 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "kv_store_centralized_mooncake": "/inclusionAI/Ling-3.0-flash/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/inclusionAI/Ling-3.0-flash/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/inclusionAI/Ling-3.0-flash/hw/h200.json", "h100": "/inclusionAI/Ling-3.0-flash/hw/h100.json", "b200": "/inclusionAI/Ling-3.0-flash/hw/b200.json", "gb200": "/inclusionAI/Ling-3.0-flash/hw/gb200.json", "b300": "/inclusionAI/Ling-3.0-flash/hw/b300.json", "gb300": "/inclusionAI/Ling-3.0-flash/hw/gb300.json", "dgx_spark_gb10": "/inclusionAI/Ling-3.0-flash/hw/dgx_spark_gb10.json", "mi300x": "/inclusionAI/Ling-3.0-flash/hw/mi300x.json", "mi325x": "/inclusionAI/Ling-3.0-flash/hw/mi325x.json", "mi350x": "/inclusionAI/Ling-3.0-flash/hw/mi350x.json", "mi355x": "/inclusionAI/Ling-3.0-flash/hw/mi355x.json" } }, "model": { "model_id": "inclusionAI/Ling-3.0-flash", "min_vllm_version": "0.28.0", "architecture": "moe", "parameter_count": "124B", "active_parameters": "5.5B", "context_length": 262144, "base_args": [ "--trust-remote-code", "--dtype", "bfloat16", "--gpu-memory-utilization", "0.9", "--enable-chunked-prefill", "--compilation-config", "{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--enable-prefix-caching" ], "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable Ling 3 automatic tool calling", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "ling3" ] }, "reasoning": { "description": "Split Ling 3 thinking traces into reasoning", "args": [ "--reasoning-parser", "ling3" ] }, "spec_decoding": { "description": "Speculative decoding with the built-in MTP head or the external Ling DSpark draft model", "default_mode": "mtp", "modes": { "mtp": { "label": "MTP", "description": "Use the target checkpoint's native MTP head with three speculative tokens.", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" ], "hardware_overrides": { "mi355x": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}", "--attention-backend", "TRITON_MLA" ] } } }, "dspark": { "label": "DSpark", "description": "Use inclusionAI/Ling-3.0-flash-dspark as an external draft model with any Ling-3.0-flash target variant. K may be 1-8; this recipe recommends K7.", "args": [ "--speculative-config", "{\"method\":\"dspark\",\"model\":\"inclusionAI/Ling-3.0-flash-dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"FLASH_ATTN\",\"enable_adaptive_verification\":false}" ] } } } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 300, "description": "BF16 weights, the reference precision." }, "fp8": { "model_id": "inclusionAI/Ling-3.0-flash-fp8", "precision": "fp8", "vram_minimum_gb": 150, "tp": 2, "description": "Block-FP8 (E4M3) weights, 128x128 block scaling.", "json": "/inclusionAI/Ling-3.0-flash-fp8.json" }, "fp4": { "model_id": "inclusionAI/Ling-3.0-flash-fp4", "precision": "fp4", "precision_hardware": {}, "vram_minimum_gb": 85, "tp": 1, "description": "Block-FP8 dense with MXFP4 routed experts. Emulated on gfx942 โ€” prefer INT4 there.", "json": "/inclusionAI/Ling-3.0-flash-fp4.json" }, "int4": { "model_id": "inclusionAI/Ling-3.0-flash-int4", "precision": "int4", "vram_minimum_gb": 95, "tp": 1, "description": "compressed-tensors W4 (group 32) on the routed experts.", "json": "/inclusionAI/Ling-3.0-flash-int4.json" } }, "hardware_overrides": { "amd": { "extra_env": { "VLLM_ROCM_USE_AITER": "1" } } }, "guide": "## Overview\n\n`inclusionAI/Ling-3.0-flash` uses the `BailingMoeV3ForCausalLM` architecture\nwith a hybrid MLA/KDA attention stack, 512 routed experts (8 active per token),\none shared expert, and a native multi-token prediction head. The 42-layer base\nmodel has 124.4B total and 5.5B active parameters. The checkpoint also contains\na 3.1B MTP layer, bringing the complete checkpoint to 127.5B parameters. A\nserialized block-FP8 checkpoint is available as\n`inclusionAI/Ling-3.0-flash-fp8`. Lower-memory checkpoints are also available\nas `inclusionAI/Ling-3.0-flash-fp4` and\n`inclusionAI/Ling-3.0-flash-int4`. The external\n`inclusionAI/Ling-3.0-flash-dspark` draft can accelerate any target variant\nwithout changing which target checkpoint is served.\n\n## Prerequisites\n\n- **vLLM:** 0.28.0 or newer, which includes native Bailing V3 support;\n- **Validated hardware:** NVIDIA H20, H20-3e, H200, DGX Spark (FP4/INT4), and AMD MI300X / MI355X\n- **Precision:** BF16, serialized block FP8, mixed block-FP8/MXFP4, or compressed-tensors INT4 weights with BF16 compute\n- **Context length:** 262,144 tokens\n\n## Launching the Server\n\n```bash\nNCCL_DEBUG=WARN vllm serve inclusionAI/Ling-3.0-flash \\\n --trust-remote-code \\\n --dtype bfloat16 \\\n --tensor-parallel-size 4 \\\n --gpu-memory-utilization 0.9 \\\n --enable-chunked-prefill \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser ling3 \\\n --reasoning-parser ling3 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}'\n```\n\nFor the serialized FP8 variant on 2x H200, use:\n\n```bash\nvllm serve inclusionAI/Ling-3.0-flash-fp8 \\\n --trust-remote-code \\\n --dtype bfloat16 \\\n --tensor-parallel-size 2 \\\n --gpu-memory-utilization 0.9 \\\n --enable-chunked-prefill \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser ling3 \\\n --reasoning-parser ling3\n```\n\nFor the validated TP4+EP4 alternative on 4x H200, use:\n\n```bash\nvllm serve inclusionAI/Ling-3.0-flash-fp8 \\\n --trust-remote-code \\\n --dtype bfloat16 \\\n --tensor-parallel-size 4 \\\n --enable-expert-parallel \\\n --gpu-memory-utilization 0.9 \\\n --enable-chunked-prefill \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser ling3 \\\n --reasoning-parser ling3\n```\n\nFor the FP4 variant on one GPU, use:\n\n```bash\nvllm serve inclusionAI/Ling-3.0-flash-fp4 \\\n --trust-remote-code \\\n --dtype bfloat16 \\\n --tensor-parallel-size 1 \\\n --gpu-memory-utilization 0.9 \\\n --enable-chunked-prefill \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser ling3 \\\n --reasoning-parser ling3\n```\n\nFor the INT4 variant on one GPU, use:\n\n```bash\nvllm serve inclusionAI/Ling-3.0-flash-int4 \\\n --trust-remote-code \\\n --dtype bfloat16 \\\n --tensor-parallel-size 1 \\\n --gpu-memory-utilization 0.9 \\\n --enable-chunked-prefill \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser ling3 \\\n --reasoning-parser ling3\n```\n\n## Speculative Decoding\n\nThe **Spec decoding** control exposes MTP and DSpark in the same place. They\nare mutually exclusive modes, so the generated command contains exactly one\n`--speculative-config`.\n\n### MTP\n\nUse the target checkpoint's native MTP head with three draft tokens:\n\n```bash\n--speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":3}'\n```\n\n### DSpark\n\n`inclusionAI/Ling-3.0-flash-dspark` is an external draft model. It does not\nreplace the served target checkpoint. It can be paired with any\nLing-3.0-flash target variant; pass the DSpark repository through\n`--speculative-config.model` while keeping the selected target as the model\nserved by `vllm serve`.\n\nThe checkpoint was trained with query block size `Q=8`. Runtime\n`num_speculative_tokens` (`K`) may be any integer from 1 through 8. The recipe\nemits `K=7` as the recommended throughput/KV-capacity trade-off; change the\nvalue to select another supported width. `K=8` remains valid.\n\n```bash\n--speculative-config '{\"method\":\"dspark\",\"model\":\"inclusionAI/Ling-3.0-flash-dspark\",\"num_speculative_tokens\":7,\"draft_sample_method\":\"probabilistic\",\"attention_backend\":\"FLASH_ATTN\",\"enable_adaptive_verification\":false}'\n```\n\n`enable_adaptive_verification` is disabled because the current Ling hybrid\nMLA/KDA path is validated with fixed-width DSpark verification. The\n`attention_backend` setting applies only to the external draft model; the\ntarget model keeps its automatically selected attention backends.\n\n## Single-GPU INT4\n\n`inclusionAI/Ling-3.0-flash-int4` is a `compressed-tensors` / `pack-quantized`\ncheckpoint: symmetric W4, `group_size: 32`, applied to the **routed experts\nonly** โ€” attention, `lm_head`, the shared expert and the dense projections are\nall in the config's `ignore` list. It is not covered by the model-specific\nquantization plumbing in `bailing_moe_v3.py` (which handles block FP8 and\nMXFP4); it loads through vLLM's generic `compressed-tensors` path and runs at\nTP1 on H200 or DGX Spark.\n\nThe engine names the mechanism at startup:\n\n```\nquantization=compressed-tensors ... trust_remote_code=True, dtype=torch.bfloat16\nINFO [int_wna16.py:297] Using 'MARLIN' WNA16 MoE backend.\nINFO [cuda.py:492] Using FLASH_ATTN_MLA attention backend\nINFO [int_wna16.py:409] Using MarlinExperts\n```\n\nThe vendor model card documents SGLang only, and there is no merged vLLM PR\nspecific to Ling INT4. This variant works through the generic quantization\npath; tool calling, reasoning parsing, chunked prefill and CUDA graph capture\nbehave as they do on the other variants.\n\n## Running on MI300X/MI355X\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nvllm serve inclusionAI/Ling-3.0-flash \\\n --trust-remote-code \\\n --dtype bfloat16 \\\n --tensor-parallel-size 4 \\\n --gpu-memory-utilization 0.9 \\\n --enable-chunked-prefill \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}' \\\n --enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser ling3 \\\n --reasoning-parser ling3\n```\n\nOn MI300X the BF16 run served the full 262,144-token context with a 105.15 GiB\nKV pool and 52.38x max concurrency.\n\n**MTP** needs a backend pin on gfx950: `ROCM_AITER_MLA` lets the speculator\ncapture a graph the backend cannot serve and the workers abort with a memory\naccess fault, so the mode pins `--attention-backend TRITON_MLA` on `mi355x`.\ngfx942 is unaffected and keeps AITER MLA.\n\n## Validation\n\nThe BF16, FP8, FP4, and INT4 checkpoints support NVIDIA H20 and H200. For FP8,\nTP2 remains the recommended multi-GPU default; TP4+EP4 is a validated H200\nalternative for deployments that prefer expert parallelism. DGX Spark is\nlimited to the TP1 FP4 and INT4 variants because the BF16 and FP8 checkpoints\nare too large for its 128 GB unified memory. DSpark is external and works with\nany target variant that fits the selected hardware.\n\nThe `int4` variant was verified on **1x H200 (SM90), TP=1**, on a `main` build\nreporting `0.26.1rc1.dev1133+gf94666b60`, serving at `--max-model-len 32768`:\n77.0 GB of weights on disk, 70.27 GiB resident after load (165 s), 5 min 56 s\nfrom container start to ready, `FULL_AND_PIECEWISE` capture (68 PIECEWISE + 68\nFULL, 2.15 GiB), and a 4,072,594-token KV pool. Single-stream decode measured\n**159.5 tok/s** at TTFT p50 0.17 s; 973 output tok/s aggregate at concurrency\n16, with 0 failed requests and 0 preemptions over a 1..24 concurrency sweep.\n\nMI355X was verified on MI350X silicon, the same gfx950 class.\n\n## Thinking Mode\n\nThinking is selected per request through the chat template rather than by a\nserver flag:\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nresponse = client.chat.completions.create(\n model=\"inclusionAI/Ling-3.0-flash\",\n messages=[{\"role\": \"user\", \"content\": \"Solve the problem step by step.\"}],\n temperature=0.0,\n max_tokens=200000,\n extra_body={\"chat_template_kwargs\": {\"enable_thinking\": True}},\n)\nmsg = response.choices[0].message\nprint(msg.reasoning)\nprint(msg.content)\n```\n\nWhen serving a quantized variant, set `model` in the client request to the\ncorresponding FP8, FP4, or INT4 checkpoint ID.\n\n## References\n\n- [Ling-3.0-flash on Hugging Face](https://huggingface.co/inclusionAI/Ling-3.0-flash)\n- [Ling-3.0-flash-fp8 on Hugging Face](https://huggingface.co/inclusionAI/Ling-3.0-flash-fp8)\n- [Ling-3.0-flash-dspark on Hugging Face](https://huggingface.co/inclusionAI/Ling-3.0-flash-dspark)\n- [Ling-3.0-flash-fp4 on Hugging Face](https://huggingface.co/inclusionAI/Ling-3.0-flash-fp4)\n- [Ling-3.0-flash-int4 on Hugging Face](https://huggingface.co/inclusionAI/Ling-3.0-flash-int4)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/inclusionAI/Ling-3.0-flash.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/inclusionAI/Ling-3.0-flash.yaml", "overlays": [] } }