{ "hf_id": "tencent/Hy3", "meta": { "title": "Hy3", "slug": "hy3", "provider": "Hy (Tencent)", "description": "Tencent Hy3 — scaled-up MoE language model (295B total / 21B active) with a 3.8B MTP layer for speculative decoding, 256K context, and hy_v3 tool/reasoning parsers", "date_added": "2026-07-06", "date_updated": "2026-07-06", "difficulty": "intermediate", "tasks": [ "text" ], "performance_headline": "Hy3 MoE — 295B/21B on 8×H200, 8×H20-3e(141GB), or 8×AMD MI300X/MI355X with MTP", "related_recipes": [ "tencent/Hy3-preview" ], "hardware": { "gb300": "verified", "mi300x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" }, "docker_image": "vllm/vllm-openai:hy3", "command": "vllm serve tencent/Hy3 \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --reasoning-parser hy_v3", "argv": [ "vllm", "serve", "tencent/Hy3", "--tensor-parallel-size", "8", "--tool-call-parser", "hy_v3", "--enable-auto-tool-choice", "--reasoning-parser", "hy_v3" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm \\\n vllm/vllm-openai:hy3 tencent/Hy3 \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --reasoning-parser hy_v3", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm", "vllm/vllm-openai:hy3", "tencent/Hy3", "--tensor-parallel-size", "8", "--tool-call-parser", "hy_v3", "--enable-auto-tool-choice", "--reasoning-parser", "hy_v3" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/tencent/Hy3/strategies/single_node_tep.json", "single_node_dep": "/tencent/Hy3/strategies/single_node_dep.json", "multi_node_tp": "/tencent/Hy3/strategies/multi_node_tp.json", "multi_node_tep": "/tencent/Hy3/strategies/multi_node_tep.json", "multi_node_dep": "/tencent/Hy3/strategies/multi_node_dep.json", "kv_store_centralized_mooncake": "/tencent/Hy3/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/tencent/Hy3/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/tencent/Hy3/hw/h200.json", "h100": "/tencent/Hy3/hw/h100.json", "b200": "/tencent/Hy3/hw/b200.json", "gb200": "/tencent/Hy3/hw/gb200.json", "b300": "/tencent/Hy3/hw/b300.json", "gb300": "/tencent/Hy3/hw/gb300.json", "mi300x": "/tencent/Hy3/hw/mi300x.json", "mi325x": "/tencent/Hy3/hw/mi325x.json", "mi355x": "/tencent/Hy3/hw/mi355x.json" } }, "model": { "model_id": "tencent/Hy3", "min_vllm_version": "0.26.0", "docker_image": { "nvidia": "vllm/vllm-openai:hy3", "amd": "vllm/vllm-openai-rocm:nightly-cbe9c40f998f13975b967773ac7e7920e115387f" }, "nightly_required": true, "architecture": "moe", "parameter_count": "295B", "active_parameters": "21B", "context_length": 262144, "base_args": [], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --pre \\\n --extra-index-url https://wheels.vllm.ai/nightly/cu130 \\\n --extra-index-url https://download.pytorch.org/whl/cu130 \\\n --index-strategy unsafe-best-match", "note": "Hy3's latest optimizations (vLLM #47433) aren't in a stable release yet — use nightly wheels or build from source." }, "docker": { "command": "docker pull vllm/vllm-openai:hy3", "note": "Use the dedicated hy3 image until changes land in vllm:latest." } } }, "features": { "tool_calling": { "description": "Hy v3 tool call parser with automatic tool choice", "args": [ "--tool-call-parser", "hy_v3", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "Hy v3 reasoning parser for thinking-mode chain-of-thought extraction", "args": [ "--reasoning-parser", "hy_v3" ] }, "spec_decoding": { "description": "Multi-Token Prediction speculative decoding using the model's built-in MTP layer", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":2}" ] } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 708, "description": "Full precision BF16 — 8×H200 or 8×H20-3e(141GB) minimum for weights + KV cache" }, "fp8": { "model_id": "tencent/Hy3-FP8", "precision": "fp8", "vram_minimum_gb": 354, "description": "FP8-quantized instruct checkpoint (tencent/Hy3-FP8) — fits a single 8×H200 / 8×H20-3e node with wide headroom.", "hardware_overrides": { "amd": { "extra_args": [ "--kv-cache-dtype", "fp8" ] } }, "json": "/tencent/Hy3-FP8.json" }, "nvfp4": { "model_id": "RedHatAI/Hy3-NVFP4-FP8", "precision": "nvfp4", "vram_minimum_gb": 214, "description": "Mixed-precision quant by Red Hat AI — MoE experts in NVFP4, attention in FP8 block. Halves GPU requirement vs BF16 (fits 4×H100 80GB / 4×H200).", "json": "/RedHatAI/Hy3-NVFP4-FP8.json" } }, "hardware_overrides": { "hopper": { "extra_env": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" } }, "blackwell": { "extra_env": { "VLLM_FLASHINFER_ALLREDUCE_BACKEND": "trtllm" } }, "amd": { "extra_args": [ "--attention-backend", "ROCM_AITER_FA" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "guide": "# Hy3 Usage Guide\n\nHy3 is Tencent's latest open-source Mixture-of-Experts language model:\n295B total parameters with 21B activated per token, plus a 3.8B MTP layer for\nspeculative decoding. 80 transformer layers, 192 routed experts (top-8) + 1 shared\nexpert, GQA with 64 heads over 8 KV heads, 256K context.\n\n## Availability\n\nHy3 uses the same model code as\n[Hy3-preview](https://huggingface.co/tencent/Hy3-preview), so any vLLM build that\nserves Hy3-preview also serves Hy3. The latest optimizations — Tencent's HPC-Ops\nattention and MoE backends — require a newer vLLM that includes\n[#47433](https://github.com/vllm-project/vllm/pull/47433) plus the separately-built\nHPC-Ops kernels (see the HPC-Ops backends section below).\n\n## Setup\n\nChoose one of the following setup methods.\n\n### Using Docker\n\n```bash\n# Switch to the trtllm all-reduce backend to work around an mnnvl workspace-size issue.\ndocker run --gpus all \\\n -p 8000:8000 \\\n --ipc=host \\\n -e VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:hy3 tencent/Hy3 \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3\n```\n\n### Installing from source\n\n```bash\nuv venv --python 3.12 --seed --managed-python\nsource .venv/bin/activate\ngit clone https://github.com/vllm-project/vllm.git\ncd vllm\nuv pip install --editable . --torch-backend=auto\n```\n\n## Model Deployment\n\nTo serve Hy3 on 8 GPUs, use H20-3e(141GB), H200, AMD MI300X/MI325X (192 GB),\nAMD MI350X/MI355X (288 GB), or other GPUs with larger memory capacity. Smaller-memory\n8-GPU configurations (8×H100 80GB, 8×A100 80GB) do not fit the BF16 weights plus KV\ncache — use multi-node TP for those.\n\n### Serving on 8×AMD MI300X / MI325X / MI350X / MI355X\n\nServe with the standard launcher plus the AITER environment variables (the recipe's\n`hardware_overrides.amd.extra_env` applies these automatically when the AMD profile is\nselected on the recipe site):\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\n# MoE disabled (CK GEMM crash); Triton fallback. See troubleshooting note below.\nexport VLLM_ROCM_USE_AITER_MOE=0\nexport VLLM_ROCM_USE_AITER_MHA=1\nexport VLLM_ROCM_USE_AITER_RMSNORM=1\nexport VLLM_ROCM_USE_AITER_LINEAR=1\n\nvllm serve tencent/Hy3 \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3 \\\n --gpu-memory-utilization 0.90\n```\n\n> **ROCm troubleshooting — AITER fused-MoE.** With `VLLM_ROCM_USE_AITER_MOE=1`\n> the engine currently aborts during warmup with\n> `RuntimeError: wrong! device_gemm with the specified compilation parameters\n> does not support this GEMM problem` (raised from `aiter` `ck_moe_stage2`).\n> Workaround: set\n> `VLLM_ROCM_USE_AITER_MOE=0` to fall back to the Triton MoE kernel; the other\n> AITER backends (MHA / RMSNorm / Linear) run as-is. The docker run needs the\n> ROCm device flags `--device /dev/kfd --device /dev/dri --group-add video\n> --group-add render` (the recipe's NVIDIA `docker run` above is CUDA-only).\n\nMTP (recommended on AMD for lower latency, same flags as the NVIDIA path):\n\n```bash\nvllm serve tencent/Hy3 \\\n --tensor-parallel-size 8 \\\n --speculative-config.method mtp \\\n --speculative-config.num_speculative_tokens 2 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3 \\\n --gpu-memory-utilization 0.90\n```\n\n### Serving on 8×H200 or 8×H20-3e(141GB)\n\nWithout Multi-Token Prediction (MTP):\n\n```bash\n# Switch to the trtllm all-reduce backend to work around an mnnvl workspace-size issue.\nexport VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm\nvllm serve tencent/Hy3 \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3\n```\n\nWith MTP (recommended for lower latency):\n\n```bash\n# Switch to the trtllm all-reduce backend to work around an mnnvl workspace-size issue.\nexport VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm\nvllm serve tencent/Hy3 \\\n --tensor-parallel-size 8 \\\n --speculative-config.method mtp \\\n --speculative-config.num_speculative_tokens 2 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3\n```\n\n### HPC-Ops backends (Hopper) — maximum throughput\n\nTencent's HPC-Ops attention and MoE kernels\n([#47433](https://github.com/vllm-project/vllm/pull/47433)) reduce latency for\nmixed-length decode and small-batch MoE on Hopper GPUs. They ship as a separate wheel —\nbuild and install it first:\n\n```bash\ngit clone https://github.com/Tencent/hpc-ops.git\ncd hpc-ops\nmake wheel\npython3 -m pip install dist/*.whl\n```\n\nEnable the attention backend with `--attention-backend HPC_ATTN`:\n\n```bash\n# Switch to the trtllm all-reduce backend to work around an mnnvl workspace-size issue.\nexport VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm\nvllm serve tencent/Hy3 \\\n --tensor-parallel-size 8 \\\n --attention-backend HPC_ATTN\n```\n\nEnable the fused MoE backend with `--moe-backend hpc`. On the FP8 checkpoint, add the\nFP8 KV-cache flags `--kv-cache-dtype fp8_e4m3 --block-size 64`:\n\n```bash\n# Switch to the trtllm all-reduce backend to work around an mnnvl workspace-size issue.\nexport VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm\nvllm serve tencent/Hy3-FP8 \\\n --tensor-parallel-size 8 \\\n --moe-backend hpc \\\n --kv-cache-dtype fp8_e4m3 \\\n --block-size 64\n```\n\nBoth backends combine with the standard `--tool-call-parser` / `--reasoning-parser` and\nMTP flags shown above. See the\n[vLLM × HPC-Ops post](https://vllm.ai/blog/2026-07-06-vllm-hpc-ops) for benchmarks and\ndetails.\n\n## Sampling and Reasoning Modes\n\nTencent's recommended sampling parameters: `temperature=0.9`, `top_p=1.0`.\n\nReasoning is controlled via `chat_template_kwargs.reasoning_effort`:\n\n| Value | Behavior |\n|:---|:---|\n| `no_think` (default) | Direct response, no chain-of-thought |\n| `low` | Light reasoning |\n| `high` | Deep chain-of-thought for math/coding/complex reasoning |\n\nWhen tools are registered, set `interleaved_thinking: true` to allow the model to\nthink between tool calls.\n\n### OpenAI Client Example\n\n```bash\nuv pip install -U openai\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nmessages = [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": \"Hello.\"},\n]\n\n# Direct response (default).\nresp = client.chat.completions.create(\n model=\"hy3\",\n messages=messages,\n temperature=0.9,\n top_p=1.0,\n max_tokens=4096,\n)\nprint(resp.choices[0].message.content)\n\n# Deep reasoning: set reasoning_effort (and interleaved_thinking if using tools).\nresp_think = client.chat.completions.create(\n model=\"hy3\",\n messages=messages,\n temperature=0.9,\n top_p=1.0,\n max_tokens=4096,\n extra_body={\n \"chat_template_kwargs\": {\n \"reasoning_effort\": \"high\",\n \"interleaved_thinking\": True,\n },\n },\n)\noutput_msg = resp_think.choices[0].message\nprint(output_msg.reasoning) # chain-of-thought\nprint(output_msg.content) # final answer\n```\n\n### cURL Usage\n\n```bash\ncurl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"hy3\",\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": \"Hello.\"}\n ],\n \"temperature\": 0.9,\n \"top_p\": 1.0,\n \"max_tokens\": 4096\n }'\n```\n\n## Benchmarking\n\nFor benchmarking, disable prefix caching by adding `--no-enable-prefix-caching` to\nthe server command.\n\nThe following benchmarks the FP8 checkpoint (8192→1024, concurrency 32, 160 prompts):\n\n```bash\nvllm bench serve \\\n --model tencent/Hy3-FP8 \\\n --dataset-name random \\\n --random-input-len 8192 \\\n --random-output-len 1024 \\\n --max-concurrency 32 \\\n --num-prompts 160 \\\n --served-model-name hy3-fp8\n```\n\nRepresentative output — Hy3-FP8, TP4 on 4×GB300 (one NVL4 tray), trtllm all-reduce,\nMTP=2:\n\n```shell\n============ Serving Benchmark Result ============\nSuccessful requests: 160\nFailed requests: 0\nMaximum request concurrency: 32\nBenchmark duration (s): 175.37\nRequest throughput (req/s): 0.91\nOutput token throughput (tok/s): 934.26\nPeak output token throughput (tok/s): 1344.00\nTotal token throughput (tok/s): 8408.34\nMean TTFT (ms): 2352.29\nMedian TTFT (ms): 505.64\nP99 TTFT (ms): 14538.85\nMean TPOT (ms): 30.11\nMedian TPOT (ms): 30.86\nP99 TPOT (ms): 40.48\nMean ITL (ms): 36.26\nMedian ITL (ms): 24.09\n==================================================\n```\n\n## References\n\n- [Hugging Face model card](https://huggingface.co/tencent/Hy3)\n- [FP8 checkpoint — tencent/Hy3-FP8](https://huggingface.co/tencent/Hy3-FP8)\n- [NVFP4-FP8 checkpoint — RedHatAI/Hy3-NVFP4-FP8](https://huggingface.co/RedHatAI/Hy3-NVFP4-FP8)\n- [GitHub: Tencent-Hunyuan/Hy3](https://github.com/Tencent-Hunyuan/Hy3)\n- [vLLM × HPC-Ops: high-performance attention & MoE backends](https://vllm.ai/blog/2026-07-06-vllm-hpc-ops)\n- [Tencent Hy3](https://hy.tencent.com/research/hy3)\n" }