Tune Qwen medium prefill batching

2026-08-30 18:39:17 +02:00
parent 6af25cf30c
commit edb8b2868f
3 changed files with 6 additions and 2 deletions
@@ -23,6 +23,8 @@ VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf

 FAST_CONTEXT=76800
 MEDIUM_CONTEXT=160000
+MEDIUM_BATCH_SIZE=2048
+MEDIUM_UBATCH_SIZE=128
 LARGE_CONTEXT=192000
 ULTRA_CONTEXT=262144
 UNCENSORED_CONTEXT=80000
@@ -181,9 +181,9 @@ services:
      - --threads-batch
      - "${LLAMA_THREADS_BATCH:-6}"
      - --batch-size
-      - "64"
+      - "${MEDIUM_BATCH_SIZE:-2048}"
      - --ubatch-size
-      - "32"
+      - "${MEDIUM_UBATCH_SIZE:-128}"
      - --parallel
      - "1"
      - --jinja
@@ -77,6 +77,8 @@ VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c239

 FAST_CONTEXT=76800
 MEDIUM_CONTEXT=160000
+MEDIUM_BATCH_SIZE=2048
+MEDIUM_UBATCH_SIZE=128
 MEDIUM_TENSOR_SPLIT=90,10
 LARGE_CONTEXT=192000
 LARGE_TENSOR_SPLIT=86,14