michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Tune prefill batching across Qwen profiles

Browse Source
This commit is contained in:
Mikei386
2026-09-15 16:57:53 +02:00
parent 78055996f5
commit b48fcc63d3
5 changed files with 37 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose.yaml
+4 -4
View File
@@ -105,9 +105,9 @@ services:
- --threads-batch
- "${LLAMA_THREADS_BATCH:-6}"
- --batch-size
- "${FAST_BATCH_SIZE:-64}"
- "${FAST_BATCH_SIZE:-2048}"
- --ubatch-size
- "${FAST_UBATCH_SIZE:-32}"
- "${FAST_UBATCH_SIZE:-64}"
- --parallel
- "${FAST_PARALLEL_SLOTS:-1}"
- --kv-unified
@@ -269,7 +269,7 @@ services:
- --batch-size
- "${LARGE_BATCH_SIZE:-2048}"
- --ubatch-size
- "${LARGE_UBATCH_SIZE:-128}"
- "${LARGE_UBATCH_SIZE:-256}"
- --parallel
- "${LARGE_PARALLEL_SLOTS:-1}"
- --kv-unified
@@ -431,7 +431,7 @@ services:
- --batch-size
- "${UNCENSORED_BATCH_SIZE:-2048}"
- --ubatch-size
- "${UNCENSORED_UBATCH_SIZE:-128}"
- "${UNCENSORED_UBATCH_SIZE:-256}"
- --parallel
- "${UNCENSORED_PARALLEL_SLOTS:-1}"
- --kv-unified