michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Increase llama prompt cache to 32 GiB

Browse Source
This commit is contained in:
Mikei386
2026-09-03 10:03:43 +02:00
parent 4153e535d3
commit fb0cb40bed
4 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose.yaml
+5 -5
View File
@@ -104,7 +104,7 @@ services:
# disabled until the current upstream restore regressions are fixed.
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -179,7 +179,7 @@ services:
- q4_0
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -264,7 +264,7 @@ services:
- q4_0
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -342,7 +342,7 @@ services:
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -424,7 +424,7 @@ services:
- q4_0
- --cache-prompt
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-24576}"
- "${LLAMA_CACHE_RAM_MIB:-32768}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch