michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tune prefill batching across Qwen profiles

Browse Source
This commit is contained in:
Mikei386
2026-08-30 22:11:26 +02:00
parent 9c10b0ab88
commit 27d2bb1b4b
4 changed files with 46 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/install.env.example
+8
View File
@@ -76,15 +76,23 @@ VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c239
# draft-model artifact is neither downloaded nor passed to llama-server.
FAST_CONTEXT=76800
FAST_BATCH_SIZE=64
FAST_UBATCH_SIZE=32
MEDIUM_CONTEXT=160000
MEDIUM_BATCH_SIZE=2048
MEDIUM_UBATCH_SIZE=128
MEDIUM_TENSOR_SPLIT=90,10
LARGE_CONTEXT=192000
LARGE_BATCH_SIZE=2048
LARGE_UBATCH_SIZE=128
LARGE_TENSOR_SPLIT=86,14
ULTRA_CONTEXT=262144
ULTRA_BATCH_SIZE=2048
ULTRA_UBATCH_SIZE=128
ULTRA_TENSOR_SPLIT=80,20
UNCENSORED_CONTEXT=80000
UNCENSORED_BATCH_SIZE=2048
UNCENSORED_UBATCH_SIZE=128
UNCENSORED_TENSOR_SPLIT=90,10
UNCENSORED_MTP_MAX=2
EXPERIMENTAL_CONTEXT=76800