Tune prefill batching across Qwen profiles

2026-08-30 19:50:40 +02:00
parent edb8b2868f
commit 2661553e20
4 changed files with 46 additions and 8 deletions
@@ -22,12 +22,20 @@ EXPERIMENTAL_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf
 VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf

 FAST_CONTEXT=76800
+FAST_BATCH_SIZE=64
+FAST_UBATCH_SIZE=32
 MEDIUM_CONTEXT=160000
 MEDIUM_BATCH_SIZE=2048
 MEDIUM_UBATCH_SIZE=128
 LARGE_CONTEXT=192000
+LARGE_BATCH_SIZE=2048
+LARGE_UBATCH_SIZE=128
 ULTRA_CONTEXT=262144
+ULTRA_BATCH_SIZE=2048
+ULTRA_UBATCH_SIZE=128
 UNCENSORED_CONTEXT=80000
+UNCENSORED_BATCH_SIZE=2048
+UNCENSORED_UBATCH_SIZE=128
 EXPERIMENTAL_CONTEXT=76800
 FAST_GPU_DEVICES=0,1
 MEDIUM_GPU_DEVICES=0,1