Increase medium prefill micro-batch

2026-09-15 16:36:48 +02:00
parent 54d38a31aa
commit 78055996f5
3 changed files with 3 additions and 3 deletions
@@ -183,7 +183,7 @@ services:
      - --batch-size
      - "${MEDIUM_BATCH_SIZE:-2048}"
      - --ubatch-size
-      - "${MEDIUM_UBATCH_SIZE:-128}"
+      - "${MEDIUM_UBATCH_SIZE:-512}"
      - --parallel
      - "${MEDIUM_PARALLEL_SLOTS:-1}"
      - --kv-unified