Tune prefill batching across Qwen profiles

2026-08-30 19:50:40 +02:00
parent edb8b2868f
commit 2661553e20
4 changed files with 46 additions and 8 deletions
@@ -76,15 +76,23 @@ VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c239
 # draft-model artifact is neither downloaded nor passed to llama-server.

 FAST_CONTEXT=76800
+FAST_BATCH_SIZE=64
+FAST_UBATCH_SIZE=32
 MEDIUM_CONTEXT=160000
 MEDIUM_BATCH_SIZE=2048
 MEDIUM_UBATCH_SIZE=128
 MEDIUM_TENSOR_SPLIT=90,10
 LARGE_CONTEXT=192000
+LARGE_BATCH_SIZE=2048
+LARGE_UBATCH_SIZE=128
 LARGE_TENSOR_SPLIT=86,14
 ULTRA_CONTEXT=262144
+ULTRA_BATCH_SIZE=2048
+ULTRA_UBATCH_SIZE=128
 ULTRA_TENSOR_SPLIT=80,20
 UNCENSORED_CONTEXT=80000
+UNCENSORED_BATCH_SIZE=2048
+UNCENSORED_UBATCH_SIZE=128
 UNCENSORED_TENSOR_SPLIT=90,10
 UNCENSORED_MTP_MAX=2
 EXPERIMENTAL_CONTEXT=76800