michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Increase medium prefill micro-batch

Browse Source
This commit is contained in:
Mikei386
2026-09-15 16:36:48 +02:00
parent 54d38a31aa
commit 78055996f5
3 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+1 -1
View File
@@ -26,7 +26,7 @@ FAST_BATCH_SIZE=64
FAST_UBATCH_SIZE=32 FAST_UBATCH_SIZE=32
MEDIUM_CONTEXT=160000 MEDIUM_CONTEXT=160000
MEDIUM_BATCH_SIZE=2048 MEDIUM_BATCH_SIZE=2048
MEDIUM_UBATCH_SIZE=128 MEDIUM_UBATCH_SIZE=512
LARGE_CONTEXT=192000 LARGE_CONTEXT=192000
LARGE_BATCH_SIZE=2048 LARGE_BATCH_SIZE=2048
LARGE_UBATCH_SIZE=128 LARGE_UBATCH_SIZE=128
compose.yaml
+1 -1
View File
@@ -183,7 +183,7 @@ services:
- --batch-size - --batch-size
- "${MEDIUM_BATCH_SIZE:-2048}" - "${MEDIUM_BATCH_SIZE:-2048}"
- --ubatch-size - --ubatch-size
- "${MEDIUM_UBATCH_SIZE:-128}" - "${MEDIUM_UBATCH_SIZE:-512}"
- --parallel - --parallel
- "${MEDIUM_PARALLEL_SLOTS:-1}" - "${MEDIUM_PARALLEL_SLOTS:-1}"
- --kv-unified - --kv-unified
config/install.env.example
+1 -1
View File
@@ -84,7 +84,7 @@ FAST_BATCH_SIZE=64
FAST_UBATCH_SIZE=32 FAST_UBATCH_SIZE=32
MEDIUM_CONTEXT=160000 MEDIUM_CONTEXT=160000
MEDIUM_BATCH_SIZE=2048 MEDIUM_BATCH_SIZE=2048
MEDIUM_UBATCH_SIZE=128 MEDIUM_UBATCH_SIZE=512
MEDIUM_TENSOR_SPLIT=85,15 MEDIUM_TENSOR_SPLIT=85,15
LARGE_CONTEXT=192000 LARGE_CONTEXT=192000
LARGE_BATCH_SIZE=2048 LARGE_BATCH_SIZE=2048