michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable cross-chat llama prompt cache reuse

Browse Source
This commit is contained in:
Mikei386
2026-08-25 19:19:49 +02:00
parent 57afdd4f15
commit d921dfaf69
7 changed files with 60 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose.yaml
+33
View File
@@ -90,6 +90,14 @@ services:
- q4_0
- --cache-type-v
- q4_0
# Keep the cross-chat prefix cache explicit. cache-reuse tolerates small
# changes after Hermes' stable system-prompt prefix without enabling the
# currently unreliable on-disk slot restore path.
- --cache-prompt
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -163,6 +171,11 @@ services:
- q4_0
- --cache-type-v
- q4_0
- --cache-prompt
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -247,6 +260,11 @@ services:
- q4_0
- --cache-type-v
- q4_0
- --cache-prompt
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -319,6 +337,11 @@ services:
- q4_0
- --cache-type-v
- q4_0
- --cache-prompt
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -397,6 +420,11 @@ services:
- q4_0
- --cache-type-v
- q4_0
- --cache-prompt
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- --threads
- "${LLAMA_THREADS:-6}"
- --threads-batch
@@ -469,6 +497,11 @@ services:
- q4_0
- --cache-type-v
- q4_0
- --cache-prompt
- --cache-reuse
- "${LLAMA_CACHE_REUSE:-256}"
- --cache-ram
- "${LLAMA_CACHE_RAM_MIB:-8192}"
- --parallel
- "1"
- --jinja