michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Increase llama prompt cache to 32 GiB

Browse Source
This commit is contained in:
Mikei386
2026-09-03 10:03:43 +02:00
parent 4153e535d3
commit fb0cb40bed
4 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+1 -1
View File
@@ -49,7 +49,7 @@ UNCENSORED_MTP_MAX=2
LLAMA_THREADS=6
LLAMA_THREADS_BATCH=6
FAST_PARALLEL_SLOTS=1
LLAMA_CACHE_RAM_MIB=24576
LLAMA_CACHE_RAM_MIB=32768
MEDIUM_PARALLEL_SLOTS=1
LARGE_PARALLEL_SLOTS=1
ULTRA_PARALLEL_SLOTS=1
compose.yaml
+5 -5
View File
@@ -104,7 +104,7 @@ services:
# disabled until the current upstream restore regressions are fixed.
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -179,7 +179,7 @@ services:
- q4_0
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -264,7 +264,7 @@ services:
- q4_0
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -342,7 +342,7 @@ services:
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -424,7 +424,7 @@ services:
- q4_0
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
config/install.env.example
+1 -1
View File
@@ -98,7 +98,7 @@ UNCENSORED_MTP_MAX=2
LLAMA_THREADS=6
LLAMA_THREADS_BATCH=6
FAST_PARALLEL_SLOTS=1
LLAMA_CACHE_RAM_MIB=24576
LLAMA_CACHE_RAM_MIB=32768
MEDIUM_PARALLEL_SLOTS=1
LARGE_PARALLEL_SLOTS=1
ULTRA_PARALLEL_SLOTS=1
install.sh
+1 -1
View File
@@ -358,7 +358,7 @@ IMAGE_GPU_DEVICES=${IMAGE_GPU_DEVICES:-${TEXT_GPU_DEVICES:-0}}
FLUX_MODEL_DIR=${FLUX_MODEL_DIR:-/data/models/FLUX.2-klein-4B}
LLAMA_THREADS=${LLAMA_THREADS:-6}
LLAMA_THREADS_BATCH=${LLAMA_THREADS_BATCH:-6}
LLAMA_CACHE_RAM_MIB=${LLAMA_CACHE_RAM_MIB:-24576}
LLAMA_CACHE_RAM_MIB=${LLAMA_CACHE_RAM_MIB:-32768}
DEFAULT_REASONING_EFFORT=${DEFAULT_REASONING_EFFORT:-off}
EOF
chmod 0600 $SECRETS_DIR/stack.env