{ "hf_id": "mistralai/Mistral-Medium-3.5-128B", "meta": { "title": "Mistral-Medium-3.5", "slug": "mistral-medium-3.5", "provider": "Mistral AI", "description": "Mistral Medium 3.5 (128B) dense vision-language model with native FP8 weights and 256K context", "date_added": "2026-04-30", "date_updated": "2026-05-26", "tasks": [ "multimodal" ], "related_recipes": [ "mistralai/Mistral-Large-3-675B-Instruct-2512", "mistralai/Ministral-3-8B-Reasoning-2512" ], "hardware": { "mi300x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve mistralai/Mistral-Medium-3.5-128B \\\n --tokenizer_mode mistral \\\n --config_format mistral \\\n --load_format mistral \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral \\\n --reasoning-parser mistral", "argv": [ "vllm", "serve", "mistralai/Mistral-Medium-3.5-128B", "--tokenizer_mode", "mistral", "--config_format", "mistral", "--load_format", "mistral", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "mistral", "--reasoning-parser", "mistral" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest mistralai/Mistral-Medium-3.5-128B \\\n --tokenizer_mode mistral \\\n --config_format mistral \\\n --load_format mistral \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral \\\n --reasoning-parser mistral", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "mistralai/Mistral-Medium-3.5-128B", "--tokenizer_mode", "mistral", "--config_format", "mistral", "--load_format", "mistral", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "mistral", "--reasoning-parser", "mistral" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "multi_node_tp": "/mistralai/Mistral-Medium-3.5-128B/strategies/multi_node_tp.json", "kv_store_centralized_mooncake": "/mistralai/Mistral-Medium-3.5-128B/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/mistralai/Mistral-Medium-3.5-128B/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/mistralai/Mistral-Medium-3.5-128B/hw/h200.json", "h100": "/mistralai/Mistral-Medium-3.5-128B/hw/h100.json", "b200": "/mistralai/Mistral-Medium-3.5-128B/hw/b200.json", "gb200": "/mistralai/Mistral-Medium-3.5-128B/hw/gb200.json", "b300": "/mistralai/Mistral-Medium-3.5-128B/hw/b300.json", "gb300": "/mistralai/Mistral-Medium-3.5-128B/hw/gb300.json", "mi300x": "/mistralai/Mistral-Medium-3.5-128B/hw/mi300x.json", "mi325x": "/mistralai/Mistral-Medium-3.5-128B/hw/mi325x.json", "mi350x": "/mistralai/Mistral-Medium-3.5-128B/hw/mi350x.json", "mi355x": "/mistralai/Mistral-Medium-3.5-128B/hw/mi355x.json" } }, "model": { "model_id": "mistralai/Mistral-Medium-3.5-128B", "min_vllm_version": "0.18.0", "architecture": "dense", "parameter_count": "128B", "active_parameters": "128B", "context_length": 262144, "base_args": [ "--tokenizer_mode", "mistral", "--config_format", "mistral", "--load_format", "mistral" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" }, "extras": [ { "note": "Mistral tokenizer / chat-template runtime — Mistral 3.5 needs >= 1.11.1 (auto-installed by vLLM nightly, pin if you hit an older cached wheel)", "command": "uv pip install -U \"mistral_common>=1.11.1\"" } ] } }, "features": { "tool_calling": { "description": "Mistral tool-call parser with automatic tool choice", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "mistral" ] }, "reasoning": { "description": "Mistral reasoning parser extracts [THINK]...[/THINK] into message.reasoning (emitted when reasoning_effort='high')", "args": [ "--reasoning-parser", "mistral" ] }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "encoder_parallel": { "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.", "args": [ "--mm-encoder-tp-mode", "data" ] }, "spec_decoding": { "description": "EAGLE speculative decoding via mistralai/Mistral-Medium-3.5-128B-EAGLE draft head", "args": [ "--speculative-config", "{\"model\":\"mistralai/Mistral-Medium-3.5-128B-EAGLE\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}" ] } }, "opt_in_features": [ "text_only", "encoder_parallel", "spec_decoding" ], "variants": { "default": { "precision": "fp8", "vram_minimum_gb": 154, "description": "Native FP8 weights (vision tower / projector / lm_head kept in BF16); recommended on 8xH200, 8xMI300X, or 4xB200" } }, "kv_offload_support": { "offloading_cpu": "verified", "offloading_fs": "verified" }, "hardware_overrides": { "amd": { "extra_args": [ "--no-enable-prefix-caching" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "guide": "## Overview\n\nMistral-Medium-3.5 is a 128B dense vision-language model from Mistral AI. The\nweights ship pre-quantized to FP8 (E4M3) with the vision tower, multimodal\nprojector, and `lm_head` retained in BF16. Image input is supported up to\n1540x1540 (Pixtral-style encoder, patch size 14). Context length is 256K\nvia YaRN scaling (factor 64x over the 4K base).\n\nReasoning is opt-in per request via `reasoning_effort: \"high\"` — when set,\nthe model emits `[THINK]...[/THINK]` blocks that the Mistral reasoning\nparser surfaces as `message.reasoning`. Tool calling uses the\n`[AVAILABLE_TOOLS]` / `[TOOL_CALLS]` chat-template tokens.\n\n## Prerequisites\n\n- Hardware: 8xH200 (recommended), 4xB200 or 2-8xMI300X; single B200 / MI300X also fits the\n weights (~134 GB raw) but leaves little room for the 256K KV cache - see below.\n- vLLM **nightly** (Mistral 3.5 architecture support has not yet shipped in a\n stable release).\n\n### Install vLLM\n\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend=auto \\\n --extra-index-url https://wheels.vllm.ai/nightly\n```\n\nThis pulls in `mistral_common >= 1.11.1` and `transformers >= 5.4.0` automatically.\n\n## Launch command\n\n8xH200 (or 8xB200):\n\n```bash\nvllm serve mistralai/Mistral-Medium-3.5-128B \\\n --tensor-parallel-size 8 \\\n --tokenizer_mode mistral --config_format mistral --load_format mistral \\\n --enable-auto-tool-choice --tool-call-parser mistral \\\n --reasoning-parser mistral\n```\n\nUseful flags:\n\n- `--max-model-len`: default 262144; lower it (e.g. 65536) to free VRAM for\n larger batch sizes on tighter GPU pools.\n- `--language-model-only`: skip the vision encoder entirely for text-only\n workloads.\n- `--mm-encoder-tp-mode data`: run the small vision encoder data-parallel\n instead of tensor-parallel — avoids the all-reduce overhead.\n- `--limit-mm-per-prompt.image N`: cap images per request.\n\n## EAGLE speculative decoding\n\nMistral ships a dedicated EAGLE draft head at\n[`mistralai/Mistral-Medium-3.5-128B-EAGLE`](https://huggingface.co/mistralai/Mistral-Medium-3.5-128B-EAGLE).\nIt is **not** included in the default config — toggle the `spec_decoding` feature.\n\nMistral's recommended serve command (from the EAGLE model card):\n\n```bash\nvllm serve mistralai/Mistral-Medium-3.5-128B --tensor-parallel-size 8 \\\n --tool-call-parser mistral --enable-auto-tool-choice --reasoning-parser mistral \\\n --max-num-batched-tokens 16384 --max-num-seqs 128 --gpu-memory-utilization 0.8 \\\n --speculative-config '{\"model\":\"mistralai/Mistral-Medium-3.5-128B-EAGLE\",\"num_speculative_tokens\":3,\"method\":\"eagle\",\"max_model_len\":\"65536\"}'\n```\n\nThe draft model is a 2-layer Mistral-style head trained on the 128B target;\nit shares the tokenizer and runs at TP=8 alongside the target.\n\n## Client usage\n\nReasoning + tool calling against the OpenAI-compatible endpoint:\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\nresp = client.chat.completions.create(\n model=\"mistralai/Mistral-Medium-3.5-128B\",\n messages=[{\"role\": \"user\", \"content\": \"Plan a 3-day Paris trip.\"}],\n extra_body={\"reasoning_effort\": \"high\"},\n temperature=0.7, max_tokens=4096,\n)\nmsg = resp.choices[0].message\nprint(\"reasoning:\", msg.reasoning)\nprint(\"answer:\", msg.content)\n```\n\nImage input (vision):\n\n```python\nresp = client.chat.completions.create(\n model=\"mistralai/Mistral-Medium-3.5-128B\",\n messages=[{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://...\"}},\n {\"type\": \"text\", \"text\": \"Describe this image.\"},\n ],\n }],\n max_tokens=512,\n)\n```\n\n### AMD MI300X (ROCm)\n\nThe command builder fits the model on one GPU on MI300X. This is prone to fail due to OOM. Use the following instead.\n\n**Single GPU** - Use `--tensor-parallel-size 1` and limit context, e.g.\n`--max-model-len 131072`. Without that limit, KV-cache allocation fails at the\ndefault **262144** context even in text-only mode. You can also reduce\n`--max-num-batched-tokens` or change `--gpu-memory-utilization` if you still hit OOM.\nHere is a text-only example:\n\n```bash\ndocker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e SAFETENSORS_FAST_GPU=1 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-rocm:nightly mistralai/Mistral-Medium-3.5-128B \\\n --tokenizer_mode mistral \\\n --config_format mistral \\\n --load_format mistral \\\n --tensor-parallel-size 1 \\\n --no-enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral \\\n --reasoning-parser mistral \\\n --max-model-len 131072 \\\n --language-model-only\n```\n\n**Text-only** - Set `--tensor-parallel-size` to the number of GPUs you wish to use:\n\n```bash\ndocker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -e SAFETENSORS_FAST_GPU=1 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-rocm:nightly mistralai/Mistral-Medium-3.5-128B \\\n --tokenizer_mode mistral \\\n --config_format mistral \\\n --load_format mistral \\\n --tensor-parallel-size 8 \\\n --no-enable-prefix-caching \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral \\\n --reasoning-parser mistral \\\n --language-model-only\n```\n\nFor `lm_eval` against a text-only server, pass `fix_mistral_regex=True` in\n`--model_args` (Mistral 3.5 tokenizer quirk).\n\n**Multimodal (text + image)** - Enable the image inputs and set the limit for images per prompt, here 1, as necessary:\n\n```bash\ndocker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -e SAFETENSORS_FAST_GPU=1 \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-rocm:nightly mistralai/Mistral-Medium-3.5-128B \\\n --tokenizer_mode mistral --config_format mistral --load_format mistral \\\n --tensor-parallel-size 8 \\\n --no-enable-prefix-caching \\\n --enable-mm \\\n --limit-mm-per-prompt '{\"image\": 1}' \\\n --enable-auto-tool-choice --tool-call-parser mistral \\\n --reasoning-parser mistral\n```\n\n### MI300X benchmarking (text-only)\n\nServing benchmark: `vllm bench serve` with 100 requests, max concurrency 32,\n1024 input + 1024 output tokens per request.\nAccuracy: `lm_eval` GSM8k (5-shot), `local-completions` backend.\n\n| TP | Output tok/s | Mean TTFT (ms) | Mean TPOT (ms) | GSM8k flexible | GSM8k strict |\n|----|-------------:|---------------:|---------------:|---------------:|-------------:|\n| 1 | 438 | 5719 | 55.2 | 0.928 | 0.876 |\n| 2 | 530 | 5310 | 45.0 | 0.923 | 0.867 |\n| 4 | 729 | 3317 | 33.0 | 0.928 | 0.873 |\n| 8 | 915 | 2522 | 26.1 | 0.929 | 0.876 |\n\n## Troubleshooting\n\n- **OOM at full 256K context** on H200 or MI300X: drop `--max-model-len` to\n **131072** or **65536**, or set `--language-model-only` if you don't need vision.\n- `reasoning_effort` rejected: only `\"none\"` and `\"high\"` are accepted by the\n chat template — anything else raises an exception.\n\n## References\n\n- [Model card](https://huggingface.co/mistralai/Mistral-Medium-3.5-128B)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/mistralai/Mistral-Medium-3.5-128B.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/mistralai/Mistral-Medium-3.5-128B.yaml", "overlays": [] } }