{ "hf_id": "mistralai/Ministral-3-14B-Instruct-2512", "meta": { "title": "Ministral-3-Instruct", "slug": "ministral-3-instruct", "provider": "Mistral AI", "description": "Ministral 3 Instruct family (3B/8B/14B) with FP8 weights, vision support, and 256K context", "date_added": "2025-12-02", "date_updated": "2026-05-25", "tasks": [ "multimodal" ], "related_recipes": [ "mistralai/Ministral-3-8B-Reasoning-2512", "mistralai/Mistral-Large-3-675B-Instruct-2512" ], "hardware": { "h200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve mistralai/Ministral-3-14B-Instruct-2512 \\\n --tokenizer_mode mistral \\\n --config_format mistral \\\n --load_format mistral \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral", "argv": [ "vllm", "serve", "mistralai/Ministral-3-14B-Instruct-2512", "--tokenizer_mode", "mistral", "--config_format", "mistral", "--load_format", "mistral", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "mistral" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest mistralai/Ministral-3-14B-Instruct-2512 \\\n --tokenizer_mode mistral \\\n --config_format mistral \\\n --load_format mistral \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "mistralai/Ministral-3-14B-Instruct-2512", "--tokenizer_mode", "mistral", "--config_format", "mistral", "--load_format", "mistral", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "mistral" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "multi_node_tp": "/mistralai/Ministral-3-14B-Instruct-2512/strategies/multi_node_tp.json", "kv_store_centralized_mooncake": "/mistralai/Ministral-3-14B-Instruct-2512/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/mistralai/Ministral-3-14B-Instruct-2512/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/mistralai/Ministral-3-14B-Instruct-2512/hw/h200.json", "h100": "/mistralai/Ministral-3-14B-Instruct-2512/hw/h100.json", "b200": "/mistralai/Ministral-3-14B-Instruct-2512/hw/b200.json", "gb200": "/mistralai/Ministral-3-14B-Instruct-2512/hw/gb200.json", "b300": "/mistralai/Ministral-3-14B-Instruct-2512/hw/b300.json", "gb300": "/mistralai/Ministral-3-14B-Instruct-2512/hw/gb300.json", "mi300x": "/mistralai/Ministral-3-14B-Instruct-2512/hw/mi300x.json", "mi325x": "/mistralai/Ministral-3-14B-Instruct-2512/hw/mi325x.json", "mi350x": "/mistralai/Ministral-3-14B-Instruct-2512/hw/mi350x.json", "mi355x": "/mistralai/Ministral-3-14B-Instruct-2512/hw/mi355x.json" } }, "model": { "model_id": "mistralai/Ministral-3-14B-Instruct-2512", "min_vllm_version": "0.11.0", "architecture": "dense", "parameter_count": "14B", "active_parameters": "14B", "context_length": 262144, "base_args": [ "--tokenizer_mode", "mistral", "--config_format", "mistral", "--load_format", "mistral" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Mistral tool-call parser with automatic tool choice", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "mistral" ] }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "encoder_parallel": { "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.", "args": [ "--mm-encoder-tp-mode", "data" ] } }, "opt_in_features": [ "text_only", "encoder_parallel" ], "variants": { "default": { "precision": "fp8", "vram_minimum_gb": 17, "description": "Native FP8 weights (14B), fits on 1x H200" }, "8b": { "model_id": "mistralai/Ministral-3-8B-Instruct-2512", "precision": "fp8", "vram_minimum_gb": 12, "description": "Smaller 8B variant with independent embedding/output layers", "json": "/mistralai/Ministral-3-8B-Instruct-2512.json" }, "3b": { "model_id": "mistralai/Ministral-3-3B-Instruct-2512", "precision": "fp8", "vram_minimum_gb": 6, "description": "Smallest 3B variant with tied embeddings", "json": "/mistralai/Ministral-3-3B-Instruct-2512.json" }, "fp8": { "model_id": "mistralai/Ministral-3-14B-Instruct-2512-FP8", "precision": "fp8", "vram_minimum_gb": 17, "description": "FP8 quantized weights for Hopper/Blackwell GPUs", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/mistralai/Ministral-3-14B-Instruct-2512-FP8.json" }, "redhat_fp8": { "model_id": "RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic", "precision": "fp8", "label": "FP8 Dynamic (RedHat)", "vram_minimum_gb": 17, "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Ministral-3-14B-Instruct-2512-FP8-dynamic.json" }, "nvfp4": { "model_id": "RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 9, "description": "RedHatAI NVFP4 quantization", "json": "/RedHatAI/Ministral-3-14B-Instruct-2512-NVFP4.json" } }, "kv_offload_support": { "offloading_cpu": "verified", "offloading_fs": "verified" }, "hardware_overrides": { "amd": { "extra_args": [ "--config_format", "mistral", "--load_format", "mistral" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1" } } }, "guide": "## Overview\n\nMinistral-3 Instruct comes with FP8 weights in 3 different sizes:\n\n- 3B: tied embeddings (shares embedding and output layers)\n- 8B and 14B: independent embedding and output layers\n\nEach variant has vision support and a 256K context length. Smaller models offer faster\ninference at the cost of lower quality; pick the best trade-off for your use case.\n\n## Prerequisites\n\n- Hardware: 1x H200 (sufficient for all three sizes thanks to FP8 weights);\n 1x MI300X (verified) / MI325X / MI355X\n- vLLM >= 0.11.0\n\n### Install vLLM\n\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto\n```\n\n## Launch command\n\n```bash\nvllm serve mistralai/Ministral-3-14B-Instruct-2512 \\\n --tokenizer_mode mistral --config_format mistral --load_format mistral \\\n --enable-auto-tool-choice --tool-call-parser mistral\n```\n\nFor 8B: `mistralai/Ministral-3-8B-Instruct-2512`\nFor 3B: `mistralai/Ministral-3-3B-Instruct-2512`\n\n- `enable-auto-tool-choice`: required for tool usage\n- `tool-call-parser mistral`: required for tool usage\n- `--max-model-len` defaults to `262144`; reduce to save memory\n- `--max-num-batched-tokens` balances throughput and latency\n\n### AMD (MI300X / MI325X / MI355X)\n\nVerified on an 8-GPU MI300X node with **TP=1** per variant.\n\n**3B**\n\n```bash\ndocker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-rocm:latest mistralai/Ministral-3-3B-Instruct-2512 \\\n --tokenizer_mode mistral \\\n --tensor-parallel-size 1 \\\n --config_format mistral \\\n --load_format mistral \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral\n```\n\n**8B**\n\n```bash\ndocker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-rocm:latest mistralai/Ministral-3-8B-Instruct-2512 \\\n --tokenizer_mode mistral \\\n --tensor-parallel-size 1 \\\n --config_format mistral \\\n --load_format mistral \\\n --max-model-len auto \\\n --max-num-batched-tokens 8192 \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral\n```\n\n**14B**\n\n```bash\ndocker run --device=/dev/kfd --device=/dev/dri \\\n --security-opt seccomp=unconfined --group-add video \\\n --privileged --ipc=host -p 8000:8000 \\\n -e VLLM_ROCM_USE_AITER=1 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-rocm:latest mistralai/Ministral-3-14B-Instruct-2512 \\\n --tokenizer_mode mistral \\\n --tensor-parallel-size 1 \\\n --config_format mistral \\\n --load_format mistral \\\n --max-num-seqs 256 \\\n --max-model-len auto \\\n --gpu-memory-utilization 0.95 \\\n --max-num-batched-tokens 8192 \\\n --enable-auto-tool-choice \\\n --tool-call-parser mistral\n```\n\n## Client Usage\n\nVision reasoning example:\n\n```python\nfrom datetime import datetime, timedelta\nfrom openai import OpenAI\nfrom huggingface_hub import hf_hub_download\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\nmodel = client.models.list().data[0].id\n\ndef load_system_prompt(repo_id, filename):\n path = hf_hub_download(repo_id=repo_id, filename=filename)\n with open(path) as f:\n prompt = f.read()\n today = datetime.today().strftime(\"%Y-%m-%d\")\n yesterday = (datetime.today() - timedelta(days=1)).strftime(\"%Y-%m-%d\")\n return prompt.format(name=repo_id.split(\"/\")[-1], today=today, yesterday=yesterday)\n\nSYSTEM_PROMPT = load_system_prompt(model, \"SYSTEM_PROMPT.txt\")\nimage_url = \"https://static.wikia.nocookie.net/essentialsdocs/images/7/70/Battle.png/revision/latest?cb=20220523172438\"\n\nresponse = client.chat.completions.create(\n model=model,\n messages=[\n {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": [\n {\"type\": \"text\", \"text\": \"What action should I take here?\"},\n {\"type\": \"image_url\", \"image_url\": {\"url\": image_url}},\n ]},\n ],\n temperature=0.15, max_tokens=262144,\n)\nprint(response.choices[0].message.content)\n```\n\nFunction calling and text-only examples follow a similar OpenAI-compatible pattern.\n\n## Benchmarking (MI300X verification)\n\nServing benchmarks used `vllm bench serve` with random 1024-token input/output,\n`--max-concurrency 32`, and `--num-prompts 100` against each variant above.\nAccuracy used `lm-eval` GSM8K (5-shot, `flexible-extract` / `strict-match` filters).\n\n### Throughput\n\n| Variant | Output tok/s | Mean TTFT (ms) | Mean TPOT (ms) |\n|---------|-------------:|---------------:|---------------:|\n| 3B | 3842 | 288 | 6.42 |\n| 8B | 2468 | 1117 | 9.48 |\n| 14B | 1941 | 1229 | 12.22 |\n\n### GSM8K (5-shot `exact_match`)\n\n| Variant | flexible-extract | strict-match |\n|---------|-----------------:|-------------:|\n| 3B | 0.7786 ± 0.0114 | 0.7445 ± 0.0120 |\n| 8B | 0.8560 ± 0.0097 | 0.8491 ± 0.0099 |\n| 14B | 0.8795 ± 0.0090 | 0.8764 ± 0.0091 |\n\n**14B** full `vllm bench serve` output (TP=1, MI300X):\n\n```shell\n============ Serving Benchmark Result ============\nSuccessful requests: 100\nFailed requests: 0\nMaximum request concurrency: 32\nBenchmark duration (s): 52.76\nTotal input tokens: 102400\nTotal generated tokens: 102400\nRequest throughput (req/s): 1.90\nOutput token throughput (tok/s): 1940.79\nPeak output token throughput (tok/s): 3126.00\nPeak concurrent requests: 64.00\nTotal token throughput (tok/s): 3881.58\n---------------Time to First Token----------------\nMean TTFT (ms): 1228.72\nMedian TTFT (ms): 952.25\nP99 TTFT (ms): 2925.25\n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 12.22\nMedian TPOT (ms): 12.25\nP99 TPOT (ms): 12.83\n---------------Inter-token Latency----------------\nMean ITL (ms): 12.22\nMedian ITL (ms): 11.78\nP99 ITL (ms): 13.66\n==================================================\n```\n\n## Troubleshooting\n\n- OOM: lower `--max-model-len` (e.g. 32768) or use the 3B/8B variant.\n\n## References\n\n- [Ministral-3-14B-Instruct](https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512)\n- [Ministral-3-8B-Instruct](https://huggingface.co/mistralai/Ministral-3-8B-Instruct-2512)\n- [Ministral-3-3B-Instruct](https://huggingface.co/mistralai/Ministral-3-3B-Instruct-2512)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "564ac0de1b228568cafcbfbe8ccaccc82d6023eb", "path": "recipes/models/mistralai/Ministral-3-14B-Instruct-2512.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/mistralai/Ministral-3-14B-Instruct-2512.yaml", "overlays": [] } }