{ "hf_id": "zai-org/GLM-5.3-Flash", "meta": { "title": "GLM-5.3-Flash", "slug": "glm-5.3-flash", "provider": "GLM (Z-AI)", "description": "GLM-5.3-Flash is a 320B-total / 18B-active multimodal MoE with hybrid KDA and sparse MLA attention, native FP8 weights, MTP, and a 1M-token context window.", "date_added": "2026-08-27", "date_updated": "2026-10-19", "tasks": [ "text", "multimodal" ], "performance_headline": "320B MoE with 18B active parameters and 1M-token context", "hardware": { "h100": "verified", "b200": "verified", "gb200": "verified", "mi355x": "verified", "ascend_950pr_8x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_ENGINE_READY_TIMEOUT_S": "3600", "VLLM_USE_RUST_FRONTEND": "1" }, "docker_image": "vllm/vllm-openai:nightly", "command": "vllm serve zai-org/GLM-5.3-Flash \\\n --tokenizer-mode glm5_next \\\n --tensor-parallel-size 8 \\\n --disable-flashinfer-autotune \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --reasoning-parser glm47", "argv": [ "vllm", "serve", "zai-org/GLM-5.3-Flash", "--tokenizer-mode", "glm5_next", "--tensor-parallel-size", "8", "--disable-flashinfer-autotune", "--tool-call-parser", "glm47", "--enable-auto-tool-choice", "--reasoning-parser", "glm47" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ENGINE_READY_TIMEOUT_S=3600 \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n vllm/vllm-openai:nightly zai-org/GLM-5.3-Flash \\\n --tokenizer-mode glm5_next \\\n --tensor-parallel-size 8 \\\n --disable-flashinfer-autotune \\\n --tool-call-parser glm47 \\\n --enable-auto-tool-choice \\\n --reasoning-parser glm47", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ENGINE_READY_TIMEOUT_S=3600", "-e", "VLLM_USE_RUST_FRONTEND=1", "vllm/vllm-openai:nightly", "zai-org/GLM-5.3-Flash", "--tokenizer-mode", "glm5_next", "--tensor-parallel-size", "8", "--disable-flashinfer-autotune", "--tool-call-parser", "glm47", "--enable-auto-tool-choice", "--reasoning-parser", "glm47" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/zai-org/GLM-5.3-Flash/strategies/single_node_tep.json", "single_node_dep": "/zai-org/GLM-5.3-Flash/strategies/single_node_dep.json", "multi_node_tp": "/zai-org/GLM-5.3-Flash/strategies/multi_node_tp.json", "multi_node_tp_pp": "/zai-org/GLM-5.3-Flash/strategies/multi_node_tp_pp.json", "multi_node_tep": "/zai-org/GLM-5.3-Flash/strategies/multi_node_tep.json", "multi_node_dep": "/zai-org/GLM-5.3-Flash/strategies/multi_node_dep.json", "pd_cluster": "/zai-org/GLM-5.3-Flash/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/zai-org/GLM-5.3-Flash/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/zai-org/GLM-5.3-Flash/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/zai-org/GLM-5.3-Flash/hw/h200.json", "h100": "/zai-org/GLM-5.3-Flash/hw/h100.json", "b200": "/zai-org/GLM-5.3-Flash/hw/b200.json", "gb200": "/zai-org/GLM-5.3-Flash/hw/gb200.json", "b300": "/zai-org/GLM-5.3-Flash/hw/b300.json", "gb300": "/zai-org/GLM-5.3-Flash/hw/gb300.json", "mi300x": "/zai-org/GLM-5.3-Flash/hw/mi300x.json", "mi325x": "/zai-org/GLM-5.3-Flash/hw/mi325x.json", "mi350x": "/zai-org/GLM-5.3-Flash/hw/mi350x.json", "mi355x": "/zai-org/GLM-5.3-Flash/hw/mi355x.json", "ascend_950pr_8x": "/zai-org/GLM-5.3-Flash/hw/ascend_950pr_8x.json" } }, "model": { "model_id": "zai-org/GLM-5.3-Flash", "min_vllm_version": "0.29.0", "nightly_required": true, "default_frontend": "rust", "docker_image": { "nvidia": "vllm/vllm-openai:nightly", "amd": "vllm/vllm-openai-rocm:nightly" }, "architecture": "moe", "parameter_count": "321B", "active_parameters": "18B", "context_length": 1048576, "base_args": [ "--tokenizer-mode", "glm5_next" ], "base_env": { "VLLM_ENGINE_READY_TIMEOUT_S": "3600" }, "install": { "docker": { "command": "docker pull vllm/vllm-openai:nightly", "note": "Native GLM5Next rendering requires an image containing vllm-project/vllm#62424, including its Rust binary. Use NVIDIA/ROCm nightly images only once they include this change. Ascend 950PR also requires the nightly-main-a5 platform support and GLM-5.3-Flash overlays vLLM#53906 and vllm-ascend#15127." } } }, "features": { "tool_calling": { "description": "GLM tool-call parsing with automatic tool choice", "args": [ "--tool-call-parser", "glm47", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "Extract reasoning content with the GLM reasoning parser", "args": [ "--reasoning-parser", "glm47" ] }, "spec_decoding": { "description": "Built-in MTP or an external DFlash2 drafter. Pick one method — only that --speculative-config is emitted.", "default_mode": "mtp", "modes": { "mtp": { "label": "MTP", "description": "Use the checkpoint's MTP layer to draft 5 tokens.", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" ] }, "dflash": { "label": "DFlash2", "description": "External DFlash2 drafter (incoai/GLM-5.3-Flash-DFlash2). method is dflash; DFlash2 is selected by the draft architecture. num_speculative_tokens must be 7 (block_size 8 minus one).", "hardware": { "amd": "unsupported" }, "args": [ "--speculative-config", "{\"method\":\"dflash\",\"model\":\"incoai/GLM-5.3-Flash-DFlash2\",\"num_speculative_tokens\":7}" ] } } } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "fp8", "vram_minimum_gb": 386, "description": "Default native FP8 checkpoint (zai-org/GLM-5.3-Flash)", "hardware_overrides": { "ascend_950pr_8x": { "docker_image": "quay.io/ascend/vllm-ascend:nightly-main-a5", "install": { "pip": false } }, "mi350x": { "extra_env": { "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1", "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA": "1" } }, "mi355x": { "extra_env": { "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1", "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA": "1" } } } }, "nvfp4": { "model_id": "RedHatAI/GLM-5.3-Flash-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 229, "description": "NVFP4 checkpoint (RedHatAI/GLM-5.3-Flash-NVFP4) — MoE experts quantized to 4-bit; requires Blackwell", "json": "/RedHatAI/GLM-5.3-Flash-NVFP4.json" }, "bf16": { "model_id": "zai-org/GLM-5.3-Flash-BF16", "precision": "bf16", "vram_minimum_gb": 772, "description": "Official BF16 checkpoint; requires substantially more GPU memory", "hardware_overrides": { "mi350x": { "extra_env": { "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1", "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA": "1" } }, "mi355x": { "extra_env": { "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1", "VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA": "1" } } }, "json": "/zai-org/GLM-5.3-Flash-BF16.json" } }, "kv_offload_support": { "offloading_cpu": "verified", "offloading_fs": "verified" }, "strategy_hardware": { "single_node_tep": { "npu": "unsupported" }, "single_node_dep": { "npu": "unsupported" }, "multi_node_tp": { "npu": "unsupported" }, "multi_node_tp_pp": { "npu": "unsupported" }, "multi_node_tep": { "npu": "unsupported" }, "multi_node_dep": { "npu": "unsupported" }, "pd_cluster": { "npu": "unsupported" } }, "hardware_overrides": { "hopper": { "extra_args": [ "--disable-flashinfer-autotune" ] }, "blackwell": { "extra_args": [ "--kv-cache-dtype", "fp8" ] }, "amd": { "extra_args": [ "--max-num-seqs", "512", "--attention-backend", "ROCM_AITER_MLA_SPARSE" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1" } }, "npu": { "extra_args": [ "--trust-remote-code", "--block-size", "512", "--max-model-len", "131072", "--max-num-seqs", "16", "--max-num-batched-tokens", "8192", "--gpu-memory-utilization", "0.90" ], "extra_env": { "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "HCCL_BUFFSIZE": "512", "HCCL_OP_EXPANSION_MODE": "AIV", "HCCL_CONNECT_TIMEOUT": "7200" } } }, "hardware_notes": { "mi350x": { "title": "Long-context indexer top-k opt-in", "body": "When serving long contexts (>=64k), add `--sparse-indexer-topk-backend aiter` for improved decode performance." }, "mi355x": { "title": "Long-context indexer top-k opt-in", "body": "When serving long contexts (>=64k), add `--sparse-indexer-topk-backend aiter` for improved decode performance." }, "ascend_950pr_8x": { "title": "vLLM Ascend A5 image required", "body": "Single-node TP8 on 8×950PR. Pin quay.io/ascend/vllm-ascend:nightly-main-a5 and overlay vLLM#53906 / vllm-ascend#15127. Do not pass --quantization ascend. --block-size 512 is required (kpool indexer). EP is not available on this interconnect." } }, "guide": "## Overview\n\nGLM-5.3-Flash is a multimodal mixture-of-experts model with approximately 321B\ntotal parameters and 18B active parameters per token. Its 45-layer language model\ncombines KDA linear-attention layers with NoPE sparse MLA layers, routes each token\nthrough 8 of 288 experts, and supports image and video inputs. The checkpoint declares\na maximum context length of 1,048,576 tokens and includes one MTP draft layer.\n\nThe implementation supports NVIDIA Hopper and newer GPUs, AMD Instinct\ngfx950 via ROCm, and Huawei Ascend 950PR (8 NPUs, one node) via vLLM\nAscend. On NVIDIA/AMD it can scale with tensor, pipeline, expert, or\ndata+expert parallelism. The Ascend path is validated as single-node TP=8\nonly.\n\n## Prerequisites\n\n- **Weights:** about 306 GiB for the default native FP8 checkpoint before runtime\n and KV-cache overhead; the BF16 variant requires roughly twice the weight memory\n- **vLLM:** an image containing [native GLM5Next support (#62424)](https://github.com/vllm-project/vllm/pull/62424), including the matching Rust frontend binary. Older images need an updated build for the commands below.\n- **FlashInfer:** 0.6.17 or newer is required for NoPE sparse MLA\n\nThe default model ID, `zai-org/GLM-5.3-Flash`, is the FP8 checkpoint. Select the BF16\nvariant to serve `zai-org/GLM-5.3-Flash-BF16`, or the NVFP4 variant to serve\n`RedHatAI/GLM-5.3-Flash-NVFP4`. Its MoE experts are quantized to 4-bit (NVFP4),\nand it requires NVIDIA Blackwell GPUs.\n\n## Frontends\n\nThe command builder defaults to **Rust** and adds `--tokenizer-mode glm5_next`.\nWith `VLLM_USE_RUST_FRONTEND=1`, this selects native GLM5Next chat rendering for\nthe GLM-5.3-Flash template, including tool calls and reasoning options. Image and\nvideo inputs are not yet supported on the Rust frontend; use the Python frontend\nfor multimodal requests.\n\nSelect **Python** in the command builder, or set `VLLM_USE_RUST_FRONTEND=0`,\nto use the Hugging Face renderer and processor on the same build. On older builds\nthat do not recognize `glm5_next`, also replace `--tokenizer-mode glm5_next` with\n`--tokenizer-mode auto`.\n\n## Launching the server\n\n### FP8 with TP4 and MTP on one GB200 tray\n\n```bash\nVLLM_USE_RUST_FRONTEND=1 vllm serve zai-org/GLM-5.3-Flash \\\n --tokenizer-mode glm5_next \\\n --tensor-parallel-size 4 \\\n --kv-cache-dtype fp8 \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5}' \\\n --tool-call-parser glm47 \\\n --reasoning-parser glm47 \\\n --enable-auto-tool-choice \\\n --served-model-name zai-org/GLM-5.3-Flash\n```\n\n### Prefill/Decode disaggregation (single node, TP4 + TP4)\n\nSplit one 8-GPU node into a prefill pool (GPUs 0-3) and a decode pool (GPUs 4-7),\nbridged by NIXL KV transfer. The KDA conv-state and KV-cache layouts must be pinned\nidentically on both pools, and MTP drafts run on both sides. On Blackwell you can add\n`--kv-cache-dtype fp8` to both pools; Hopper does not support FP8 KV cache for this\nmodel and must run BF16 KV.\n\n> Note: `num_speculative_tokens` must be the same in prefill and decode instance\n\n```bash\n# Prefill pool\nCUDA_VISIBLE_DEVICES=0,1,2,3 \\\nVLLM_SSM_CONV_STATE_LAYOUT=DS \\\nVLLM_KV_CACHE_LAYOUT=HND \\\nUCX_NET_DEVICES=all \\\nVLLM_NIXL_SIDE_CHANNEL_PORT=5557 \\\nVLLM_USE_RUST_FRONTEND=1 \\\nvllm serve zai-org/GLM-5.3-Flash \\\n --tokenizer-mode glm5_next \\\n --port 8001 \\\n --tensor-parallel-size 4 \\\n --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_producer\"}' \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5}' \\\n --compilation-config '{\"cudagraph_mm_encoder\": true}' \\\n --enforce-eager \\\n --no-disable-hybrid-kv-cache-manager \\\n --served-model-name zai-org/GLM-5.3-Flash\n\n# Decode pool\nCUDA_VISIBLE_DEVICES=4,5,6,7 \\\nVLLM_SSM_CONV_STATE_LAYOUT=DS \\\nVLLM_KV_CACHE_LAYOUT=HND \\\nUCX_NET_DEVICES=all \\\nVLLM_NIXL_SIDE_CHANNEL_PORT=5558 \\\nVLLM_USE_RUST_FRONTEND=1 \\\nvllm serve zai-org/GLM-5.3-Flash \\\n --tokenizer-mode glm5_next \\\n --port 8002 \\\n --tensor-parallel-size 4 \\\n --kv-transfer-config '{\"kv_connector\":\"NixlConnector\",\"kv_role\":\"kv_consumer\"}' \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5}' \\\n --max-num-seqs 512 \\\n --no-disable-hybrid-kv-cache-manager \\\n --served-model-name zai-org/GLM-5.3-Flash\n\n# Router\nvllm-router --policy round_robin --vllm-pd-disaggregation \\\n --prefill http://127.0.0.1:8001 \\\n --decode http://127.0.0.1:8002 \\\n --host 0.0.0.0 --port 8000 \\\n --intra-node-data-parallel-size 1\n```\n\n## Reasoning modes\n\nThinking is always on — the generation prompt opens a `` block unconditionally.\nGLM-5.3-Flash offers **three reasoning effort levels** driven by the `reasoning_effort`\nfield; the default is **`max`**:\n\n| Mode | How to request | Behavior |\n|------|----------------|----------|\n| **Think Max** (default) | omit `reasoning_effort`, or set `\"max\"` | Deepest reasoning — hard math, multi-step planning, agentic tasks. Highest token cost. |\n| **Think High** | `\"reasoning_effort\": \"high\"` | Balanced depth and latency. |\n| **Think Low** | `\"reasoning_effort\": \"low\"` | Lightest reasoning — simple Q&A, lowest latency and token cost. |\n\nThe chat template resolves effort to **`max` unless `reasoning_effort` is explicitly\n`\"low\"` or `\"high\"`** (any other value falls back to `max`), then injects\n`Reasoning Effort: Low|High|Max` into the system prompt. Pass it through\n`chat_template_kwargs` or the top-level OpenAI `reasoning_effort` field.\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\nmsgs = [{\"role\": \"user\", \"content\": \"Summarize sparse attention in one sentence.\"}]\n\n# Think Max (default) — just omit reasoning_effort\nclient.chat.completions.create(model=\"zai-org/GLM-5.3-Flash\", messages=msgs, max_tokens=4096)\n\n# Think High / Think Low — explicitly request the effort level\nclient.chat.completions.create(\n model=\"zai-org/GLM-5.3-Flash\",\n messages=msgs,\n max_tokens=4096,\n extra_body={\"chat_template_kwargs\": {\"reasoning_effort\": \"low\"}},\n)\n```\n\n```bash\ncurl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"zai-org/GLM-5.3-Flash\",\n \"messages\": [{\"role\": \"user\", \"content\": \"Summarize sparse attention in one sentence.\"}],\n \"temperature\": 1,\n \"max_tokens\": 4096,\n \"chat_template_kwargs\": {\"reasoning_effort\": \"high\"}\n }'\n```\n\n## Client usage\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\nresponse = client.chat.completions.create(\n model=\"zai-org/GLM-5.3-Flash\",\n messages=[{\"role\": \"user\", \"content\": \"Summarize sparse attention in one sentence.\"}],\n temperature=1.0,\n max_tokens=256,\n)\nprint(response.choices[0].message.content)\n```\n\n### Multimodal input (image / video)\n\nStandard OpenAI multi-part content — list `image_url` / `video_url` items alongside the\ntext. The chat template expands each into its `<|begin_of_image|>…` / `<|begin_of_video|>…`\nplaceholder tokens, and the same image token is reused for video frames (frame spans are\ndelimited by the video start/end tokens).\n\n```python\nfrom openai import OpenAI\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\n\n# Image\nclient.chat.completions.create(\n model=\"zai-org/GLM-5.3-Flash\",\n messages=[{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://example.com/image.png\"}},\n {\"type\": \"text\", \"text\": \"Describe the image.\"},\n ],\n }],\n max_tokens=512,\n)\n```\n\n```python\nfrom openai import OpenAI\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\n\n# Video\nclient.chat.completions.create(\n model=\"zai-org/GLM-5.3-Flash\",\n messages=[{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"video_url\", \"video_url\": {\"url\": \"https://example.com/clip.mp4\"}},\n {\"type\": \"text\", \"text\": \"What happens in this video?\"},\n ],\n }],\n max_tokens=512,\n)\n```\n\n```bash\n# Image\ncurl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"zai-org/GLM-5.3-Flash\",\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://example.com/image.png\"}},\n {\"type\": \"text\", \"text\": \"Describe the image.\"}\n ]\n }],\n \"max_tokens\": 512\n }'\n```\n\n```bash\n# Video\ncurl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"zai-org/GLM-5.3-Flash\",\n \"messages\": [{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"video_url\", \"video_url\": {\"url\": \"https://example.com/clip.mp4\"}},\n {\"type\": \"text\", \"text\": \"What happens in this video?\"}\n ]\n }],\n \"max_tokens\": 512\n }'\n```\n\n## Running on AMD Instinct\n\nUse a `vllm/vllm-openai-rocm:nightly` build containing #62424 for native Rust\nrendering. Model support landed in [vLLM #53906](https://github.com/vllm-project/vllm/pull/53906).\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\n# gfx950 (MI350X/MI355X) only\nexport VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1\nexport VLLM_ROCM_USE_AITER_TRITON_SPARSE_MLA=1\nVLLM_USE_RUST_FRONTEND=1 vllm serve zai-org/GLM-5.3-Flash \\\n --tokenizer-mode glm5_next \\\n --served-model-name zai-org/GLM-5.3-Flash \\\n --tensor-parallel-size 4 \\\n --max-num-seqs 512 \\\n --tool-call-parser glm47 \\\n --reasoning-parser glm47 \\\n --enable-auto-tool-choice \\\n --attention-backend ROCM_AITER_MLA_SPARSE\n```\n\nAt TP=4 the FP8 checkpoint reports a 14.92M-token KV pool and ~113.81x max\nconcurrency at 128K context; the BF16 checkpoint reports an 8.87M-token KV\npool at the same TP and context.\n\nMTP speculative decoding is supported on MI355X by vLLM builds containing\n[vLLM #55239](https://github.com/vllm-project/vllm/pull/55239), validated on 4×MI355X\nwith TP=4 and MTP5. Enable **Spec Decoding** and select **MTP** in the command builder\nto add the required configuration.\n\n### Long-context indexer top-k (gfx950)\n\nIf your traffic is mostly long context (>=64k tokens) and using gfx950 \n(MI350X/MI355X), using the topk indexer backend from AITER is recommended:\n\n```bash\nVLLM_USE_RUST_FRONTEND=1 vllm serve zai-org/GLM-5.3-Flash \\\n --tokenizer-mode glm5_next \\\n --attention-backend ROCM_AITER_MLA_SPARSE \\\n --sparse-indexer-topk-backend aiter\n```\n\n## Benchmarking\n\n```bash\nvllm bench serve \\\n --backend vllm \\\n --model zai-org/GLM-5.3-Flash \\\n --served-model-name zai-org/GLM-5.3-Flash \\\n --dataset-name random \\\n --random-input-len 8192 \\\n --random-output-len 1024 \\\n --max-concurrency 16 \\\n --num-prompts 64\n```\n\n## Troubleshooting\n\n- **Sparse-MLA initialization error:** verify that the image contains FlashInfer\n 0.6.18 or newer.\n\n## KV cache offloading\n\nOffloading requires a build containing [vllm-project/vllm#57145](https://github.com/vllm-project/vllm/pull/57145).\nValidation used [zai-org/GLM-5.3-Flash](https://huggingface.co/zai-org/GLM-5.3-Flash) on 4×H200 (TP4),\nvLLM commit `d1b4028d7e`, the Python frontend, `--enforce-eager`,\n`--max-model-len 16384`, `--max-num-seqs 16` and `--max-num-batched-tokens 8192`.\nValidation used `TieringOffloadingSpec`: 8 GiB total CPU tier, `blocks_per_chunk: 1`,\na persistent filesystem `root_dir`, eight read/write threads and `PYTHONHASHSEED=0`.\nThree CPU-tier reloads and three filesystem reloads after restarting the server passed.\nGLM returned matching final answers; its arithmetic reasoning field varied on filesystem reload.\nThe five-shot GSM8K diagnostic scored 31/32; this is a small sample, not the full benchmark.\nThese results cover this text-only configuration, not the builder's larger tier/chunk defaults or other variants.\n\n## References\n\n- [Model card](https://huggingface.co/zai-org/GLM-5.3-Flash)\n- [vLLM](https://github.com/vllm-project/vllm)\n- [ROCm MTP sparse-MLA fix](https://github.com/vllm-project/vllm/pull/55239)\n- **FlashInfer:** 0.6.17 or newer is required for NoPE sparse MLA\n- **Ascend 950PR:** native FP8 checkpoint (~306 GiB). On 950 the block\n scales are re-grouped to MXFP8 at load (weights stay 1 byte/element).\n Do not pass `--quantization ascend`. `--block-size 512` is required\n (kpool indexer). A2/A3 and the official BF16 checkpoint are not\n validated in this recipe.\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/zai-org/GLM-5.3-Flash.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/zai-org/GLM-5.3-Flash.yaml", "overlays": [ { "source": "vllm-ascend", "repo": "vllm-project/vllm-ascend", "ref": "main", "sha": "02ac27a19cb7676304a9ca6985ba56ce29664536", "path": "recipes/overlays/zai-org/GLM-5.3-Flash.yaml", "edit_url": "https://github.com/vllm-project/vllm-ascend/edit/main/recipes/overlays/zai-org/GLM-5.3-Flash.yaml" } ] } }