michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tune prefill batching across Qwen profiles

Browse Source
This commit is contained in:
Mikei386
2026-09-15 16:57:53 +02:00
parent 78055996f5
commit b48fcc63d3
5 changed files with 37 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/install.env.example
+4 -4
View File
@@ -80,15 +80,15 @@ DEFAULT_REASONING_EFFORT=off
# draft-model artifact is neither downloaded nor passed to llama-server.
FAST_CONTEXT=76800
FAST_BATCH_SIZE=64
FAST_UBATCH_SIZE=32
FAST_BATCH_SIZE=2048
FAST_UBATCH_SIZE=64
MEDIUM_CONTEXT=160000
MEDIUM_BATCH_SIZE=2048
MEDIUM_UBATCH_SIZE=512
MEDIUM_TENSOR_SPLIT=85,15
LARGE_CONTEXT=192000
LARGE_BATCH_SIZE=2048
LARGE_UBATCH_SIZE=128
LARGE_UBATCH_SIZE=256
LARGE_TENSOR_SPLIT=86,14
ULTRA_CONTEXT=262144
ULTRA_BATCH_SIZE=2048
@@ -96,7 +96,7 @@ ULTRA_UBATCH_SIZE=128
ULTRA_TENSOR_SPLIT=80,20
UNCENSORED_CONTEXT=80000
UNCENSORED_BATCH_SIZE=2048
UNCENSORED_UBATCH_SIZE=128
UNCENSORED_UBATCH_SIZE=256
UNCENSORED_TENSOR_SPLIT=90,10
UNCENSORED_MTP_MAX=2
LLAMA_THREADS=6