{ "hf_id": "zai-org/GLM-4.6", "meta": { "title": "GLM-4.6", "slug": "glm-4.6", "provider": "GLM (Z-AI)", "description": "GLM-4.6 MoE language model (~357B total parameters, BF16) with MTP speculative decoding, native tool calling and reasoning", "date_added": "2025-12-20", "date_updated": "2026-09-14", "tasks": [ "text" ], "performance_headline": "Updated GLM-4.X series MoE model with native FP8 and BF16, MTP speculative decoding", "related_recipes": [], "hardware": { "h200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_USE_RUST_FRONTEND": "1" }, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve zai-org/GLM-4.6 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser glm45 \\\n --enable-auto-tool-choice \\\n --reasoning-parser glm45", "argv": [ "vllm", "serve", "zai-org/GLM-4.6", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "glm45", "--enable-auto-tool-choice", "--reasoning-parser", "glm45" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n vllm/vllm-openai:latest zai-org/GLM-4.6 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser glm45 \\\n --enable-auto-tool-choice \\\n --reasoning-parser glm45", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_USE_RUST_FRONTEND=1", "vllm/vllm-openai:latest", "zai-org/GLM-4.6", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "glm45", "--enable-auto-tool-choice", "--reasoning-parser", "glm45" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/zai-org/GLM-4.6/strategies/single_node_tep.json", "single_node_dep": "/zai-org/GLM-4.6/strategies/single_node_dep.json", "multi_node_tp": "/zai-org/GLM-4.6/strategies/multi_node_tp.json", "multi_node_tp_pp": "/zai-org/GLM-4.6/strategies/multi_node_tp_pp.json", "multi_node_dep": "/zai-org/GLM-4.6/strategies/multi_node_dep.json", "multi_node_tep": "/zai-org/GLM-4.6/strategies/multi_node_tep.json", "pd_cluster": "/zai-org/GLM-4.6/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/zai-org/GLM-4.6/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/zai-org/GLM-4.6/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/zai-org/GLM-4.6/hw/h200.json", "h100": "/zai-org/GLM-4.6/hw/h100.json", "b200": "/zai-org/GLM-4.6/hw/b200.json", "gb200": "/zai-org/GLM-4.6/hw/gb200.json", "b300": "/zai-org/GLM-4.6/hw/b300.json", "gb300": "/zai-org/GLM-4.6/hw/gb300.json", "mi300x": "/zai-org/GLM-4.6/hw/mi300x.json", "mi325x": "/zai-org/GLM-4.6/hw/mi325x.json", "mi350x": "/zai-org/GLM-4.6/hw/mi350x.json", "mi355x": "/zai-org/GLM-4.6/hw/mi355x.json" } }, "model": { "model_id": "zai-org/GLM-4.6", "min_vllm_version": "0.11.0", "default_frontend": "rust", "architecture": "moe", "parameter_count": "357B", "active_parameters": "32B", "context_length": 202752, "base_args": [ "--trust-remote-code" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "GLM-4.5 tool call parser with automatic tool choice", "args": [ "--tool-call-parser", "glm45", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "GLM-4.5 reasoning parser for chain-of-thought extraction", "args": [ "--reasoning-parser", "glm45" ] }, "spec_decoding": { "description": "Multi-Token Prediction speculative decoding using the model's built-in MTP layers", "args": [ "--speculative-config.method", "mtp", "--speculative-config.num_speculative_tokens", "1" ] } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 857, "description": "Full precision BF16 on 8xH200 or equivalent" }, "fp8": { "model_id": "zai-org/GLM-4.6-FP8", "precision": "fp8", "vram_minimum_gb": 428, "description": "Native FP8 checkpoint with minimal accuracy loss — recommended for cost-efficient serving", "json": "/zai-org/GLM-4.6-FP8.json" }, "redhat_fp8": { "model_id": "RedHatAI/GLM-4.6-FP8-dynamic", "precision": "fp8", "label": "FP8 Dynamic (RedHat)", "vram_minimum_gb": 429, "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/GLM-4.6-FP8-dynamic.json" }, "redhat_fp8_plain": { "model_id": "RedHatAI/GLM-4.6-FP8", "precision": "fp8", "label": "FP8 (RedHat)", "vram_minimum_gb": 429, "description": "RedHatAI FP8 quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/GLM-4.6-FP8.json" }, "nvfp4": { "model_id": "RedHatAI/GLM-4.6-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 223, "description": "RedHatAI NVFP4 quantization", "json": "/RedHatAI/GLM-4.6-NVFP4.json" } }, "kv_offload_support": { "offloading_cpu": "verified", "offloading_fs": "verified" }, "hardware_overrides": {}, "guide": "## Overview\n\nGLM-4.6 is the successor to GLM-4.5 with ~357B total parameters. It retains the\nMoE architecture and built-in Multi-Token Prediction (MTP) layers used for\nspeculative decoding. FP8 is the recommended precision for cost-efficient\nserving with minimal accuracy loss relative to BF16.\n\n## Prerequisites\n\n- **vLLM version:** >= 0.11.0 (latest stable recommended)\n- **Hardware:** 8x H200 (BF16) or 4x-8x H200 (FP8), AMD MI300X / MI325X / MI355X for ROCm\n- **Python:** 3.10 - 3.13 (3.12 required for ROCm wheels)\n\n### Install vLLM (NVIDIA)\n\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto\n```\n\n### Install vLLM (AMD ROCm)\n\n```bash\nuv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm\n```\n\n## Launching the Server\n\n### Tensor Parallel (FP8)\n\n```bash\nvllm serve zai-org/GLM-4.6-FP8 \\\n --tensor-parallel-size 8 \\\n --tool-call-parser glm45 \\\n --reasoning-parser glm45 \\\n --enable-auto-tool-choice\n```\n\n### Enabling MTP Speculative Decoding\n\n```bash\nvllm serve zai-org/GLM-4.6-FP8 \\\n --tensor-parallel-size 4 \\\n --speculative-config.method mtp \\\n --speculative-config.num_speculative_tokens 1 \\\n --tool-call-parser glm45 \\\n --reasoning-parser glm45 \\\n --enable-auto-tool-choice\n```\n\n### Tuning Tips\n\n- `--max-model-len=65536` works well for most scenarios; max is 128K.\n- `--max-num-batched-tokens=32768` is a good default for prompt-heavy workloads.\n- `--gpu-memory-utilization=0.95` maximizes KV cache headroom.\n\n## Client Usage\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\nresp = client.chat.completions.create(\n model=\"zai-org/GLM-4.6-FP8\",\n messages=[{\"role\": \"user\", \"content\": \"Summarize MTP speculative decoding.\"}],\n max_tokens=512,\n)\nprint(resp.choices[0].message.content)\n```\n\n## Benchmarking\n\n```bash\nvllm bench serve \\\n --model zai-org/GLM-4.6-FP8 \\\n --dataset-name random \\\n --random-input-len 8000 \\\n --random-output-len 1000 \\\n --request-rate 10000 \\\n --num-prompts 16 \\\n --ignore-eos\n```\n\n## Troubleshooting\n\n- **MTP memory overhead:** Monitor GPU memory and tune batch size when enabling MTP.\n- **Tool calling not firing:** Ensure `--tool-call-parser glm45 --enable-auto-tool-choice`\n are both present.\n\n## References\n\n- [Model card](https://huggingface.co/zai-org/GLM-4.6)\n- [FP8 checkpoint](https://huggingface.co/zai-org/GLM-4.6-FP8)\n- [vLLM docs](https://docs.vllm.ai/)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/zai-org/GLM-4.6.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/zai-org/GLM-4.6.yaml", "overlays": [] } }