{ "hf_id": "baidu/ERNIE-5.5-Flash", "meta": { "title": "ERNIE-5.5-Flash", "slug": "ernie-5.5-flash", "provider": "Ernie (Baidu)", "description": "Deploy ERNIE-5.5-Flash with vLLM: Baidu's 280B-total / 14B-active Mixture-of-Experts model with hybrid HCA + MLA-DSA attention, ECHO (MTP) speculative decoding and up to 1M context, served through the vLLM-ERNIE plugin. FP8 and BF16 recipes for B300, GB300, B200, GB200 and H100, with KV cache offload and PD disaggregation.", "performance_headline": "280B / 14B-active MoE with hybrid HCA + MLA-DSA attention, up to 1M context, and ECHO (MTP) speculative decoding", "date_added": "2026-11-05", "date_updated": "2026-11-05", "tasks": [ "text" ], "related_recipes": [ "baidu/ERNIE-4.5-21B-A3B-PT" ], "hardware": { "gb200": "verified", "h100": "verified", "mi300x": "unsupported", "mi325x": "unsupported", "mi350x": "unsupported", "mi355x": "unsupported" }, "default_hardware": "gb200" }, "recommended_command": { "hardware": "gb200", "strategy": "single_node_dep", "variant": "default", "node_count": 1, "deploy_type": "single_node", "preset": "gb200_low_latency", "env": { "VLLM_ENGINE_READY_TIMEOUT_S": "3600" }, "docker_image": null, "command": "vllm serve baidu/ERNIE-5.5-Flash-Base \\\n --kv-cache-dtype fp8 \\\n --max-model-len 131072 \\\n --enable-expert-parallel \\\n --port 8100 \\\n --data-parallel-size 4 \\\n --moe-backend humming \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":2}'", "argv": [ "vllm", "serve", "baidu/ERNIE-5.5-Flash-Base", "--kv-cache-dtype", "fp8", "--max-model-len", "131072", "--enable-expert-parallel", "--port", "8100", "--data-parallel-size", "4", "--moe-backend", "humming", "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" ], "docker_command": null, "docker_argv": null, "companions": [ { "feature": "strategy:single_node_dep", "label": "vLLM Router", "after": true, "description": "Routes client requests across the DP ranks of the single engine process. --intra-node-data-parallel-size must match the engine's --data-parallel-size; consistent_hash keeps a conversation pinned to the rank that already holds its prefix. Requires: uv pip install vllm-router", "command": "vllm-router \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --policy consistent_hash \\\n --intra-node-data-parallel-size 4 \\\n --worker-urls http://localhost:8100 \\\n --worker-startup-timeout-secs 1800" } ], "strategy_spec": { "name": "single_node_dep", "deploy_type": "single_node", "display_name": "Data + Expert Parallel", "short_label": "DEP", "orientation": "throughput", "description": "Single-node DEP. Expert layers are shared across all GPUs via EP, dense layers run independently in DP groups. Best throughput for MoE models on 8-GPU single nodes. For MoE models only.\n", "hardware_match": { "min_gpus": 2, "max_gpus": 8, "multi_node": false, "architecture": "moe" }, "vllm_args": [ "--enable-expert-parallel", "--port", "8100" ], "parallel_flag": "--data-parallel-size", "companion": { "label": "vLLM Router", "description": "Routes client requests across the DP ranks of the single engine process. --intra-node-data-parallel-size must match the engine's --data-parallel-size; consistent_hash keeps a conversation pinned to the rank that already holds its prefix.", "install": "uv pip install vllm-router", "start": "after", "command": "vllm-router \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --policy consistent_hash \\\n --intra-node-data-parallel-size {dp_size} \\\n --worker-urls http://localhost:{port} \\\n --worker-startup-timeout-secs 1800\n" } }, "hardware_profile": { "brand": "NVIDIA", "generation": "blackwell", "compute_arch": "sm100", "display_name": "GB200 NVL4", "description": "NVIDIA GB200 Grace-Blackwell compute tray · 192 GB HBM3e/GPU · 4 Blackwell GPUs (NVL72 tray unit)", "gpu_count": 4, "vram_gb": 768, "multi_node": false }, "alternatives": { "single_node_tep": "/baidu/ERNIE-5.5-Flash/strategies/single_node_tep.json", "multi_node_dep": "/baidu/ERNIE-5.5-Flash/strategies/multi_node_dep.json", "pd_cluster": "/baidu/ERNIE-5.5-Flash/strategies/pd_cluster.json", "kv_store_distributed_mooncake": "/baidu/ERNIE-5.5-Flash/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "gb200": "/baidu/ERNIE-5.5-Flash/hw/gb200.json", "h100": "/baidu/ERNIE-5.5-Flash/hw/h100.json", "b200": "/baidu/ERNIE-5.5-Flash/hw/b200.json", "b300": "/baidu/ERNIE-5.5-Flash/hw/b300.json", "gb300": "/baidu/ERNIE-5.5-Flash/hw/gb300.json" } }, "model": { "model_id": "baidu/ERNIE-5.5-Flash", "min_vllm_version": "0.32.0", "architecture": "moe", "parameter_count": "280B", "active_parameters": "14B", "context_length": 1048576, "base_args": [ "--kv-cache-dtype", "fp8", "--max-model-len", "131072" ], "base_env": { "VLLM_ENGINE_READY_TIMEOUT_S": "3600" }, "install": { "pip": { "command": "uv venv --python 3.12\nsource .venv/bin/activate\nuv pip install vllm==0.32.0\nuv pip install git+https://github.com/baidu/vLLM-ERNIE.git\n", "note": "ERNIE 5.5 runs through the vLLM-ERNIE plugin, which is pinned to vLLM 0.32.0" } } }, "features": { "spec_decoding": { "description": "Speculative decoding: the checkpoint's built-in ECHO layer through vLLM's MTP path (2 draft steps by default; 1 or 3 also work), or the bundled DSpark draft of ERNIE-5.5-Flash (coming soon).", "default_mode": "mtp2", "modes": { "mtp": { "label": "MTP, 1 step", "description": "Built-in ECHO (MTP) layer, 1 draft step; mean accept length 1.98 tokens per step (GSM8K, batch size 1).", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" ] }, "mtp2": { "label": "MTP, 2 steps", "description": "The ECHO (MTP) layer runs twice per step, drafting 2 tokens; mean accept length 2.39 tokens per step (GSM8K, batch size 1).", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" ] }, "mtp3": { "label": "MTP, 3 steps", "description": "The ECHO (MTP) layer runs three times per step, drafting 3 tokens; mean accept length 2.49 tokens per step (GSM8K, batch size 1).", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" ] }, "dspark": { "label": "DSpark", "description": "DSpark draft bundled with ERNIE-5.5-Flash (coming soon); the draft length comes from the checkpoint's block size. The Base checkpoints carry no DSpark draft.", "variants": [ "flash" ], "args": [ "--speculative-config", "{\"method\":\"dspark\",\"draft_sample_method\":\"probabilistic\"}" ] } } }, "reasoning": { "description": "Split thinking into reasoning_content. For ERNIE-5.5-Flash (coming soon); the Base model has no chat training.", "args": [ "--reasoning-parser", "ernie5" ] }, "tool_calling": { "description": "Parse the model's tool-call blocks into message.tool_calls. For ERNIE-5.5-Flash (coming soon); the Base model has no chat training.", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "ernie5" ] } }, "opt_in_features": [ "spec_decoding", "reasoning", "tool_calling" ], "variants": { "default": { "model_id": "baidu/ERNIE-5.5-Flash-Base", "label": "Base · FP8", "precision": "fp8", "vram_minimum_gb": 347, "description": "ERNIE-5.5-Flash-Base, 128x128 block FP8 with UE8M0 scales; ~280 GB of weights.", "supported_hardware": [ "b300", "gb300", "b200", "gb200", "h100" ], "hardware_overrides": { "blackwell": { "extra_args": [ "--moe-backend", "humming" ] }, "h100": { "extra_args": [ "--gpu-memory-utilization", "0.93" ] } }, "json": "/baidu/ERNIE-5.5-Flash-Base.json" }, "bf16": { "model_id": "baidu/ERNIE-5.5-Flash-Base-BF16", "label": "Base · BF16", "precision": "bf16", "vram_minimum_gb": 688, "description": "ERNIE-5.5-Flash-Base in BF16; ~560 GB of weights.", "supported_hardware": [ "b300", "gb300", "b200", "gb200", "h100" ], "hardware_overrides": { "gb200": { "vram_minimum_gb": 800 }, "h100": { "extra_args": [ "--gpu-memory-utilization", "0.93" ] } }, "json": "/baidu/ERNIE-5.5-Flash-Base-BF16.json" }, "flash": { "label": "Flash (coming soon)", "precision": "fp8", "vram_minimum_gb": 347, "description": "Coming soon: the post-trained ERNIE-5.5-Flash with reasoning, tool calling and a bundled DSpark draft, in BF16, FP8 and FP4. The FP8 command is a preview until the checkpoint is published.", "supported_hardware": [ "b300", "gb300", "b200", "gb200", "h100" ], "default_modes": { "spec_decoding": "dspark" }, "hardware_overrides": { "blackwell": { "extra_args": [ "--moe-backend", "humming" ] }, "h100": { "extra_args": [ "--gpu-memory-utilization", "0.93" ] } } } }, "hardware_overrides": {}, "presets": { "gb200_low_latency": { "label": "Low-Latency", "hardware": "gb200", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "default": true, "description": "Speculative decoding on (MTP, 2 draft steps; DSpark for ERNIE-5.5-Flash). Fastest replies for a single user or light load.", "select": { "features": { "spec_decoding": true } }, "json": "/baidu/ERNIE-5.5-Flash/presets/gb200_low_latency.json" }, "gb200_high_throughput": { "label": "High-Throughput", "hardware": "gb200", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "description": "No draft. Most tokens per second across many concurrent requests.", "select": { "features": { "spec_decoding": false } }, "json": "/baidu/ERNIE-5.5-Flash/presets/gb200_high_throughput.json" }, "gb200_bf16_low_latency": { "label": "Low-Latency", "hardware": "gb200", "strategy": "multi_node_dep", "variants": [ "bf16" ], "default": true, "description": "BF16 on two GB200 trays (DP8 + EP8), speculative decoding on.", "select": { "nodes": 2, "features": { "spec_decoding": true } }, "json": "/baidu/ERNIE-5.5-Flash/presets/gb200_bf16_low_latency.json" }, "gb200_bf16_high_throughput": { "label": "High-Throughput", "hardware": "gb200", "strategy": "multi_node_dep", "variants": [ "bf16" ], "description": "BF16 on two GB200 trays (DP8 + EP8), no draft.", "select": { "nodes": 2, "features": { "spec_decoding": false } }, "json": "/baidu/ERNIE-5.5-Flash/presets/gb200_bf16_high_throughput.json" }, "b200_low_latency": { "label": "Low-Latency", "hardware": "b200", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "default": true, "description": "Speculative decoding on (MTP, 2 draft steps; DSpark for ERNIE-5.5-Flash). Fastest replies for a single user or light load.", "select": { "features": { "spec_decoding": true } }, "args": [ "--data-parallel-size", "4" ], "json": "/baidu/ERNIE-5.5-Flash/presets/b200_low_latency.json" }, "b200_high_throughput": { "label": "High-Throughput", "hardware": "b200", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "description": "No draft. Most tokens per second across many concurrent requests.", "select": { "features": { "spec_decoding": false } }, "args": [ "--data-parallel-size", "4" ], "json": "/baidu/ERNIE-5.5-Flash/presets/b200_high_throughput.json" }, "b200_bf16_low_latency": { "label": "Low-Latency", "hardware": "b200", "strategy": "single_node_dep", "variants": [ "bf16" ], "default": true, "description": "Speculative decoding on (MTP, 2 draft steps; DSpark for ERNIE-5.5-Flash). Fastest replies for a single user or light load.", "select": { "features": { "spec_decoding": true } }, "json": "/baidu/ERNIE-5.5-Flash/presets/b200_bf16_low_latency.json" }, "b200_bf16_high_throughput": { "label": "High-Throughput", "hardware": "b200", "strategy": "single_node_dep", "variants": [ "bf16" ], "description": "No draft. Most tokens per second across many concurrent requests.", "select": { "features": { "spec_decoding": false } }, "json": "/baidu/ERNIE-5.5-Flash/presets/b200_bf16_high_throughput.json" }, "gb300_low_latency": { "label": "Low-Latency", "hardware": "gb300", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "default": true, "description": "Speculative decoding on (MTP, 2 draft steps; DSpark for ERNIE-5.5-Flash). Fastest replies for a single user or light load.", "select": { "features": { "spec_decoding": true } }, "args": [ "--data-parallel-size", "2" ], "json": "/baidu/ERNIE-5.5-Flash/presets/gb300_low_latency.json" }, "gb300_high_throughput": { "label": "High-Throughput", "hardware": "gb300", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "description": "No draft. Most tokens per second across many concurrent requests.", "select": { "features": { "spec_decoding": false } }, "args": [ "--data-parallel-size", "2" ], "json": "/baidu/ERNIE-5.5-Flash/presets/gb300_high_throughput.json" }, "gb300_bf16_low_latency": { "label": "Low-Latency", "hardware": "gb300", "strategy": "single_node_dep", "variants": [ "bf16" ], "default": true, "description": "Speculative decoding on (MTP, 2 draft steps; DSpark for ERNIE-5.5-Flash). Fastest replies for a single user or light load.", "select": { "features": { "spec_decoding": true } }, "json": "/baidu/ERNIE-5.5-Flash/presets/gb300_bf16_low_latency.json" }, "gb300_bf16_high_throughput": { "label": "High-Throughput", "hardware": "gb300", "strategy": "single_node_dep", "variants": [ "bf16" ], "description": "No draft. Most tokens per second across many concurrent requests.", "select": { "features": { "spec_decoding": false } }, "json": "/baidu/ERNIE-5.5-Flash/presets/gb300_bf16_high_throughput.json" }, "b300_low_latency": { "label": "Low-Latency", "hardware": "b300", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "default": true, "description": "Speculative decoding on (MTP, 2 draft steps; DSpark for ERNIE-5.5-Flash). Fastest replies for a single user or light load.", "select": { "features": { "spec_decoding": true } }, "args": [ "--data-parallel-size", "2" ], "json": "/baidu/ERNIE-5.5-Flash/presets/b300_low_latency.json" }, "b300_high_throughput": { "label": "High-Throughput", "hardware": "b300", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "description": "No draft. Most tokens per second across many concurrent requests.", "select": { "features": { "spec_decoding": false } }, "args": [ "--data-parallel-size", "2" ], "json": "/baidu/ERNIE-5.5-Flash/presets/b300_high_throughput.json" }, "b300_bf16_low_latency": { "label": "Low-Latency", "hardware": "b300", "strategy": "single_node_dep", "variants": [ "bf16" ], "default": true, "description": "Speculative decoding on (MTP, 2 draft steps; DSpark for ERNIE-5.5-Flash). Fastest replies for a single user or light load.", "select": { "features": { "spec_decoding": true } }, "args": [ "--data-parallel-size", "4" ], "json": "/baidu/ERNIE-5.5-Flash/presets/b300_bf16_low_latency.json" }, "b300_bf16_high_throughput": { "label": "High-Throughput", "hardware": "b300", "strategy": "single_node_dep", "variants": [ "bf16" ], "description": "No draft. Most tokens per second across many concurrent requests.", "select": { "features": { "spec_decoding": false } }, "args": [ "--data-parallel-size", "4" ], "json": "/baidu/ERNIE-5.5-Flash/presets/b300_bf16_high_throughput.json" }, "h100_low_latency": { "label": "Low-Latency", "hardware": "h100", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "default": true, "description": "FP8 on one 8-GPU node (DP8 + EP8), speculative decoding on.", "select": { "features": { "spec_decoding": true } }, "json": "/baidu/ERNIE-5.5-Flash/presets/h100_low_latency.json" }, "h100_high_throughput": { "label": "High-Throughput", "hardware": "h100", "strategy": "single_node_dep", "variants": [ "default", "flash" ], "description": "FP8 on one 8-GPU node (DP8 + EP8), no draft.", "select": { "features": { "spec_decoding": false } }, "runbook": "Measured on one 8 x H100 80 GB node, DP8 + EP8, FP8 Base,\n`--gpu-memory-utilization 0.93`: 301,900 KV cache tokens per rank (2.3\nconcurrent 128K requests, or 1.15 at `--max-model-len 262144`).\nGSM8K 5-shot (full set) 94.9-96.1, and needle retrieval finds every\nneedle up to 258K tokens. At 64 concurrent requests: about 3.3k output\ntokens/s, p50 latency about 5.0 s, TTFT about 0.4 s. On Hopper the\n`humming` MoE kernel gives the same accuracy as vLLM's default, so the\nrecipe keeps the default.\n", "json": "/baidu/ERNIE-5.5-Flash/presets/h100_high_throughput.json" }, "h100_bf16_low_latency": { "label": "Low-Latency", "hardware": "h100", "strategy": "multi_node_dep", "variants": [ "bf16" ], "default": true, "description": "BF16 on two 8-GPU nodes (DP16 + EP16), speculative decoding on.", "select": { "nodes": 2, "features": { "spec_decoding": true } }, "json": "/baidu/ERNIE-5.5-Flash/presets/h100_bf16_low_latency.json" }, "h100_bf16_high_throughput": { "label": "High-Throughput", "hardware": "h100", "strategy": "multi_node_dep", "variants": [ "bf16" ], "description": "BF16 on two 8-GPU nodes (DP16 + EP16), no draft.", "select": { "nodes": 2, "features": { "spec_decoding": false } }, "json": "/baidu/ERNIE-5.5-Flash/presets/h100_bf16_high_throughput.json" }, "gb200_pd_mooncake": { "label": "1P1D Mooncake", "hardware": "gb200", "strategy": "pd_cluster", "default": true, "description": "One prefill and one decode GB200 tray (4 GPUs each), KV over Mooncake RDMA.", "select": { "pd_connector": "mooncake" }, "json_args": { "--kv-transfer-config": { "kv_connector_extra_config": { "device_name": "mlx5_0,mlx5_1,mlx5_4,mlx5_5" } } }, "roles": { "prefill": { "nodes": 1, "parallelism": "dep" }, "decode": { "nodes": 1, "parallelism": "dep", "remove_args": [ "--compilation-config" ] } }, "runbook": "Tested as one prefill and one decode GB200 tray, each DP4 + EP4, KV\nover `MooncakeConnector` on InfiniBand. The router was vllm-router\n0.1.16 (`uv pip install vllm-router==0.1.16`). With speculative\ndecoding on, both roles get the same `--speculative-config`. Replace\n`device_name` in `--kv-transfer-config` with this node's InfiniBand\nHCAs (`ibdev2netdev` lists them).\n", "json": "/baidu/ERNIE-5.5-Flash/presets/gb200_pd_mooncake.json" } }, "kv_offload_support": { "offloading_fs": "verified", "lmcache": "unsupported" }, "kv_cache_strategy_hardware": { "kv_store_centralized_mooncake": { "gb200": "unsupported", "b200": "unsupported", "gb300": "unsupported", "b300": "unsupported" } }, "strategy_hardware": { "single_node_dep": { "amd": "unsupported" }, "pd_cluster": { "amd": "unsupported" }, "multi_node_dep": { "h100": "supported", "gb200": "supported" } }, "notes": [ { "title": "ERNIE-5.5-Flash is coming soon", "when": { "variants": [ "flash" ] }, "body": "The post-trained checkpoint is not published yet. These commands show\nthe planned setup: the bundled DSpark draft, `--reasoning-parser ernie5`\nand `--tool-call-parser ernie5`.\n" }, { "title": "Parsers need the chat checkpoint", "when": { "features": [ "reasoning", "tool_calling" ], "variants": [ "default", "bf16" ] }, "body": "The Base checkpoints have no chat, reasoning or tool-call training, so\nthe parsers have nothing to parse. Use them with ERNIE-5.5-Flash.\n" }, { "title": "Speculative decoding on both PD roles", "when": { "strategies": [ "pd_cluster" ], "features": [ "spec_decoding" ] }, "body": "Prefill and decode must run the same `--speculative-config`; the\nbuilder adds it to both commands.\n" }, { "title": "Longer context on H100", "when": { "hardware": [ "h100" ] }, "body": "The commands use the 128K training length. On 80 GB GPUs\n`--max-model-len 262144` was measured to work (301,900 KV tokens per\nrank at `--gpu-memory-utilization 0.93`). The checkpoint's 1,048,576 does\nnot fit: one 1M-token request needs 16.2 GiB of KV cache per GPU, and\nvLLM refuses to start.\n" }, { "title": "1M context", "when": { "hardware": [ "b300", "gb300", "b200", "gb200" ] }, "body": "The commands use the 128K training length (`--max-model-len 131072`).\nFor the full 1M, pass `--max-model-len 1048576`: each rank's KV cache\nmust hold whole 1M-token requests, so lower `--max-num-seqs` to fit, and\na 1M-token prefill needs extra temporary memory: add\n`--gpu-memory-utilization 0.85`.\n" } ], "troubleshooting": [ { "title": "Architecture Ernie5_5_MoeForCausalLM is not supported", "body": "The vLLM-ERNIE plugin is not installed, or `VLLM_PLUGINS` excludes it.\nInstall it into the same environment as vLLM 0.32.0 and leave\n`VLLM_PLUGINS` unset (or include `ernie`).\n" }, { "title": "Timed out waiting for engine core processes to start", "body": "Loading the weights and warming up the kernels takes longer than the\ndefault timeout. Set `VLLM_ENGINE_READY_TIMEOUT_S=3600`.\n" }, { "title": "Out of memory during a 1M-token prefill", "body": "Lower `--gpu-memory-utilization` to 0.85 to leave room for the\ntemporary prefill buffers, and lower `--max-num-seqs`, or lower\n`--max-model-len`.\n" } ], "guide": "## Deployment\n\nPick your hardware, checkpoint and strategy in the builder above to\ngenerate the launch command. Every recipe runs on the recommended\n(smallest) GPU count with vLLM's defaults and the 128K training context\n(`--max-model-len 131072`); size `--max-num-seqs` and `--max-model-len` to\nyour own workload.\n\n- **Low-Latency.** ECHO speculative decoding through vLLM's MTP path (2\n draft steps; DSpark for ERNIE-5.5-Flash). Fastest replies for a single\n user or light load.\n- **High-Throughput.** Attention DP without a draft. Most tokens per\n second across many concurrent requests.\n- **PD Cluster.** One prefill and one decode GB200 tray, KV over Mooncake\n RDMA.\n\n| GPU | FP8 (~280 GB weights) | BF16 (~560 GB weights) |\n| :-- | :-- | :-- |\n| B300 / GB300 (288 GB) | 2 GPUs | 4 GPUs |\n| B200 (192 GB) | 4 GPUs | 8 GPUs |\n| GB200 (192 GB) | 4 GPUs | 2 trays × 4 GPUs |\n| H100 (80 GB) | 8 GPUs | 2 nodes × 8 GPUs |\n\nAll recipes run attention data parallel\nand the MoE expert parallel (`--data-parallel-size N --enable-expert-parallel`).\n\n## 1. Model introduction\n\n**ERNIE-5.5-Flash** is an efficient sparse Mixture-of-Experts language\nmodel with **280B total and 14B\nactivated parameters per token**, pre-trained from scratch on 33.5T tokens\nwith a training context of up to 128K. It comes as two models:\n**ERNIE-5.5-Flash-Base** (FP8 and BF16), taken at the end of pre-training\nbefore any mid- or post-training, a starting point for domain adaptation,\ninstruction tuning and reinforcement-learning research, served through the\ncompletions endpoint; and **ERNIE-5.5-Flash** (FP8 / FP4 / BF16, coming\nsoon), with reasoning, tool calling and a bundled DSpark draft.\n\nIts 43 layers are 1 dense FFN layer plus 42 Multi-Router Latent MoE layers.\nAttention is hybrid: 37 layers use Heavily Compressed Attention (HCA, one\ncached entry per 128 tokens plus a 128-token sliding window, 512-dim heads)\nand one in seven, 6 layers, keep full-context access through MLA with DSA\nsparse attention (256-dim heads). Every attention layer adds a low-rank\noutput gate, Virtual Head Attention (a low-rank post-mix across heads) and\nQK-norm. Each MoE layer has 512 routed experts (top-10) plus 1 shared expert\nin a 2048-d latent space, selected by several sigmoid routers.\nManifold-constrained hyper-connections (expansion rate 4) replace the\nresidual stream, a MultiMax head produces the logits, and ECHO reuses the\nfinal layer as a one-step speculative draft, which vLLM runs through its\nMTP path.\n\n| Model | Weights | Use |\n| :-- | :-- | :-- |\n| **ERNIE-5.5-Flash-Base** | [FP8](https://huggingface.co/baidu/ERNIE-5.5-Flash-Base) / [BF16](https://huggingface.co/baidu/ERNIE-5.5-Flash-Base-BF16) | Base model, completions. Released |\n| **ERNIE-5.5-Flash** | FP8 / FP4 / BF16 | Chat, reasoning, tool calling, DSpark. Coming soon |\n\nAll checkpoints share the architecture above: 280B total / 14B activated\nparameters.\n\nTotal parameters include the input embedding and the output head; activated\nparameters count the output head but not the embedding lookup. Both exclude\nthe 0.36B ECHO parameters. The model was trained with a context of up to\n128K tokens (131,072); fine-tuning and inference support up to 1M\n(1,048,576); pass `--max-model-len 1048576` to use it (80 GB GPUs such as\nH100 reach 262,144).\n\nThe FP8 checkpoint uses 128x128 blocks with UE8M0 scales. It keeps the\nembedding, the output head, the MoE routers, the HCA compressor, the\nindexer, the VHA post-mix in BF16, and the mHC parameters in FP32; every other linear layer and the\nrouted experts are FP8.\n\n**Sampling Defaults.** `temperature=1.0`, `top_p=0.95` (from the\ncheckpoint's `generation_config.json`), with `do_sample=false` for\nERNIE-5.5-Flash-Base and `do_sample=true` for ERNIE-5.5-Flash. vLLM reads\n`temperature` and `top_p` from it but not `do_sample`, so both models\nsample at `temperature=1.0` by default; pass `temperature=0` for greedy\ndecoding.\n\n**License.** ERNIE 5.5 Flash Base Model License Agreement v1.0 (the\n`LICENSE` file in the checkpoint).\n\nvLLM serves the model through the out-of-tree\n[vLLM-ERNIE](https://github.com/baidu/vLLM-ERNIE) plugin on the released\nvLLM 0.32.0 wheel, without modifying vLLM. Set `VLLM_USE_MODELSCOPE=True`\nto download from ModelScope; the repo names are the same on both hubs.\n\n## 2. Configuration tips\n\n- **Parallelism.** Data + Expert Parallel holds the most KV cache and gives\n the highest throughput. Tensor + Expert Parallel (TP 2, 4 or 8) lowers\n latency at low concurrency but holds less KV cache, since the single KV\n head is replicated on every TP rank.\n- **MoE Kernel.** The FP8 recipes on Blackwell pass `--moe-backend humming`,\n which supports SiTU with FP8 weights; on Hopper they keep vLLM's default.\n- **Long Context.** Every recipe passes `--max-model-len 131072`, the\n 128K training length. The checkpoint allows 1,048,576: on Blackwell pass\n `--max-model-len 1048576`, lower `--max-num-seqs` so each rank's KV\n cache holds whole 1M-token requests, and pass\n `--gpu-memory-utilization 0.85`, since a 1M-token prefill needs tens of\n GB of temporary memory on top. On 80 GB GPUs 1M does not fit and vLLM\n fails at startup; H100 reaches 262,144 at `--gpu-memory-utilization 0.93`.\n- **KV Cache.** Every recipe passes `--kv-cache-dtype fp8`, which stores\n the KV cache in FP8 and is what was tested; `auto` keeps it in BF16.\n\n## 3. Advanced usage\n\n### 3.1 Completions (Base)\n\nAs a base model, ERNIE-5.5-Flash-Base continues text: send prompts to\n`/v1/completions`. It has no chat, reasoning or tool-call training, so chat\nrequests, `--reasoning-parser` and `--tool-call-parser` do not apply. For\nfew-shot use, put the examples in the prompt and stop at the end of one\nanswer.\n\n```bash\ncurl http://localhost:8000/v1/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model\": \"baidu/ERNIE-5.5-Flash-Base\",\n \"prompt\": \"Question: What is 15% of 240?\\nAnswer:\",\n \"max_tokens\": 64,\n \"temperature\": 0,\n \"stop\": [\"\\n\"]\n }'\n```\n\nThe completion text in the response is ` 36`.\n\n### 3.2 Reasoning (ERNIE-5.5-Flash, coming soon)\n\nERNIE-5.5-Flash thinks before it answers. Start the server with\n`--reasoning-parser ernie5` (the **Reasoning** toggle above); thinking then\nlands in `reasoning_content` and the answer in `content`. Thinking is on by\ndefault; to turn it off for a request, pass `enable_thinking: false`\nthrough `chat_template_kwargs` (the chat template reads `enable_thinking`,\nnot `thinking`).\n\n```bash\n# Thinking is on by default.\ncurl http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model\": \"baidu/ERNIE-5.5-Flash\",\n \"messages\": [{\"role\": \"user\", \"content\": \"What is 15% of 240?\"}]\n }'\n\n# Turn it off for one request.\ncurl http://localhost:8000/v1/chat/completions \\\n -H 'Content-Type: application/json' \\\n -d '{\n \"model\": \"baidu/ERNIE-5.5-Flash\",\n \"messages\": [{\"role\": \"user\", \"content\": \"What is 15% of 240?\"}],\n \"chat_template_kwargs\": {\"enable_thinking\": false}\n }'\n```\n\n### 3.3 Tool calling (ERNIE-5.5-Flash, coming soon)\n\nStart the server with `--enable-auto-tool-choice --tool-call-parser ernie5`\n(the **Tool Calling** toggle above), together with\n`--reasoning-parser ernie5`. Tool calls are returned in\n`message.tool_calls`.\n\n### 3.4 KV cache offload\n\nLong shared prefixes (system prompts, agent histories) can stay outside\nGPU memory after they leave the GPU cache and be loaded back instead of\nrecomputed. Pick one on the **KV Offload** row:\n\n- **Simple.** Host memory per DP rank (`SimpleCPUOffloadConnector`).\n `cpu_bytes_to_use_per_rank` is pinned per rank, so the rank count times\n this value must stay below the node's RAM. On a GB200 tray (4 ranks) the\n default 220 GiB per rank is too much; set it to 128 GiB\n (`\"cpu_bytes_to_use_per_rank\": 137438953472`).\n- **CPU + Filesystem.** A small CPU staging tier in front of files under\n `root_dir`. Point it at fast local NVMe with plenty of space: ERNIE 5.5's\n KV blocks took about 280 KB per token on disk in our runs.\n- **Mooncake (Distributed).** Each GPU contributes host memory to a\n Mooncake pool. Install `mooncake-transfer-engine-cuda13` (vLLM 0.32.0\n ships CUDA 13), and set `MOONCAKE_DEVICE_NAME` to the InfiniBand HCAs only,\n for example `mlx5_0,mlx5_1,mlx5_4,mlx5_5` (`ibdev2netdev` lists them);\n with the empty default, Ethernet ports are picked too and transfers fail.\n\n### 3.5 Speculative decoding\n\n**ECHO (MTP).** The checkpoint carries one ECHO layer after the main stack:\nthe backbone's own final layer, trained to predict future tokens, doubles\nas the draft. vLLM runs it through its MTP path: pick **MTP, 1 / 2 / 3\nsteps** under the **Spec Decoding** toggle, which sets\n`--speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":N}'`. The\nlayer was trained for one future token and can draft several steps by\nrunning again; the Low-Latency preset uses 2 steps. Measured on GSM8K at\nbatch size 1 with greedy decoding, the mean accept length is\n1.98 / 2.39 / 2.49 tokens per step for 1 / 2 / 3 draft steps, with\nunchanged accuracy; 2 steps take most of the gain.\n\n**DSpark (ERNIE-5.5-Flash, Coming Soon).** The ERNIE-5.5-Flash checkpoint\nbundles a DSpark draft. Pick the **Flash** checkpoint above and the\n**DSpark** mode:\n`--speculative-config '{\"method\":\"dspark\",\"draft_sample_method\":\"probabilistic\"}'`.\nThe draft length comes from the checkpoint's block size, so the config\nleaves out `num_speculative_tokens`. Our runs use\n`draft_sample_method: probabilistic`; vLLM's default is `greedy`. The Base\ncheckpoints carry no DSpark draft.\n\n### 3.6 Prefill / decode disaggregation\n\nPick **PD Cluster** as the strategy. The tested setup is one prefill and one\ndecode GB200 tray with `MooncakeConnector` over RDMA, each DP4 + EP4.\nInstall `mooncake-transfer-engine-cuda13` on both trays, and set\n`device_name` in `--kv-transfer-config` to the InfiniBand HCAs only. Start\nprefill, then decode, then the router (tested with `vllm-router==0.1.16`),\nand send requests to the router. With speculative decoding, pass the same\n`--speculative-config` to the prefill and decode servers.\n\n## References\n\n- [vLLM-ERNIE plugin](https://github.com/baidu/vLLM-ERNIE)\n- [Hugging Face](https://huggingface.co/baidu/ERNIE-5.5-Flash-Base)\n- [ModelScope](https://modelscope.cn/models/baidu/ERNIE-5.5-Flash-Base)\n- [Technical report](https://ernie.baidu.com/release/ernie-5-5-flash-base/)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "c2bc949186b5cffa5c7b9dea18c973089e7f98ef", "path": "recipes/models/baidu/ERNIE-5.5-Flash.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/baidu/ERNIE-5.5-Flash.yaml", "overlays": [] } }