Add dedicated Hermes compression model

.env.example

@@ -12,6 +12,11 @@ XTTS_CACHE_DIR=/data/models/xtts-v2-cache
 XTTS_GPU_DEVICE=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
 AI_DNS=192.168.1.1
 
+# Dedicated Hermes context compressor on the RTX 3060.
+COMPRESSION_MODEL_FILE=qwen3.5-2b-compression/Qwen_Qwen3.5-2B-Q4_K_M.gguf
+COMPRESSION_CONTEXT=65536
+COMPRESSION_GPU_DEVICE=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
+
 FAST_MODEL_FILE=qwen-mix/Qwen3.8-27B-IQ4-MIX.gguf
 MEDIUM_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf
 LARGE_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf

README.md

@@ -10,6 +10,9 @@ zusammen mit dem übrigen Appdata gesichert.
 - `compose.yaml` ist der einzige Einstieg für die KI-Dienste auf Athena.
 - Genau ein llama.cpp-Profil ist aktiv. Der Router schaltet zwischen Fast,
   Medium, Large, Ultra und Uncensored.
+- Ein unabhängiges Qwen3.5-2B-Modell auf der RTX 3060 komprimiert Hermes-Chats
+  über `http://192.168.1.212:8099/v1`. Es läuft ohne Thinking mit 65K Kontext
+  und blockiert dadurch weder Router noch das aktive 27B-Hauptmodell.
 - Der **Athena Operator** bleibt als einziger hostgebundener administrativer
   MCP direkt auf Athena. MCPHub veröffentlicht seinen vorhandenen
   WireGuard-HTTP-Endpunkt zentral unter `/mcp/athena-operator`; es gibt keinen

compose.yaml

@@ -65,6 +65,75 @@ services:
       retries: 12
       start_period: 10s
 
+  # Small, non-thinking side model used only for Hermes context compression.
+  # It shares the WireGuard namespace, so Unraid can reach port 8099 while the
+  # university LAN receives no published host port. The stable GPU UUID keeps
+  # it on the RTX 3060 even if PCI enumeration changes again.
+  llama-compression:
+    image: ${LLAMA_IMAGE:-mike-ai/llama.cpp:local}
+    container_name: mike-ai-llama-compression
+    restart: unless-stopped
+    profiles: [compression]
+    gpus: all
+    network_mode: "service:wireguard-gateway"
+    depends_on:
+      wireguard-gateway:
+        condition: service_healthy
+    read_only: true
+    tmpfs:
+      - /tmp:size=256m,mode=1777
+    volumes:
+      - "${MODEL_DIR:-/srv/mike-ai/models}:/models:ro"
+    environment:
+      NVIDIA_VISIBLE_DEVICES: ${COMPRESSION_GPU_DEVICE:-GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b}
+      NVIDIA_DRIVER_CAPABILITIES: compute,utility
+    security_opt: ["no-new-privileges:true"]
+    cap_drop: [ALL]
+    command:
+      - --model
+      - "/models/${COMPRESSION_MODEL_FILE:-qwen3.5-2b-compression/Qwen_Qwen3.5-2B-Q4_K_M.gguf}"
+      - --alias
+      - qwen-compression
+      - --ctx-size
+      - "${COMPRESSION_CONTEXT:-65536}"
+      - --flash-attn
+      - "on"
+      - --cache-type-k
+      - q4_0
+      - --cache-type-v
+      - q4_0
+      - --cache-prompt
+      - --parallel
+      - "1"
+      - --jinja
+      - --host
+      - 0.0.0.0
+      - --port
+      - "8099"
+      - --metrics
+      - --n-gpu-layers
+      - all
+      - --device
+      - CUDA0
+      - --split-mode
+      - none
+      - --no-mmap
+      - --no-ui
+      - --batch-size
+      - "512"
+      - --ubatch-size
+      - "256"
+      - --threads
+      - "${LLAMA_THREADS:-6}"
+      - --threads-batch
+      - "${LLAMA_THREADS_BATCH:-6}"
+    healthcheck:
+      test: [CMD, curl, -fsS, "http://127.0.0.1:8099/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 60
+      start_period: 30s
+
   llama-fast:
     <<: *llama-common
     container_name: mike-ai-llama-fast

config/install.env.example

@@ -72,6 +72,11 @@ EXPERIMENTAL_MODEL_SHA256=40fac4050e940397dbf13087afd50f4734a11805bf9d65ef8ddd74
 VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf
 VISION_PROJECTOR_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/mmproj-BF16.gguf
 VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c2395f5c2d53
+COMPRESSION_MODEL_FILE=qwen3.5-2b-compression/Qwen_Qwen3.5-2B-Q4_K_M.gguf
+COMPRESSION_MODEL_URL=https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF/resolve/main/Qwen_Qwen3.5-2B-Q4_K_M.gguf
+COMPRESSION_MODEL_SHA256=57a1085840f497d764a7fc5d346922dbde961efb54cc792ea81d694fd846a1d8
+COMPRESSION_CONTEXT=65536
+COMPRESSION_GPU_DEVICE=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
 # All standard profiles use the MTP tensor embedded in their GGUF. A separate
 # draft-model artifact is neither downloaded nor passed to llama-server.
 

docs/STANDARD_PROFILE_MATRIX.md

@@ -30,6 +30,20 @@ und einer bewussten Aktualisierung dieser Datei.
 - Das gesonderte Experimentalprofil gehört nicht zur Benutzer-Matrix und wird
   in Open WebUI nicht als reguläres Modell angeboten.
 
+## Hermes-Kompressionsmodell
+
+Das Hilfsmodell `qwen-compression` ist kein Chatprofil. Qwen3.5-2B Q4_K_M
+läuft unabhängig mit 65.536 Token Kontext auf der RTX 3060 und ist nur über
+WireGuard auf Port 8099 erreichbar. Hermes erzwingt dafür Non-Thinking.
+
+- 30.245 Token, kalter Prompt: 4.519,7 Prefill-Tok/s.
+- Ausgabe im 30K-Qualitätstest: 117,1 Tok/s; vier von fünf ungefilterten
+  Ankern bewahrt.
+- Echter Hermes-Hilfsmodellaufruf: 158 Token in 4,6 Sekunden; alle geforderten
+  Pfade, Ports, Verbote und der Commit-Anker wurden bewahrt.
+- Zusätzlicher Speicherbedarf auf der RTX 3060: rund 1,8 GiB; nach dem Laden
+  blieben im gemessenen Produktionszustand rund 4,5 GiB frei.
+
 ## Nachweise
 
 - Fast 76,8K: synthetischer Referenzlauf, 85,50 Tok/s.

install.sh

@@ -43,7 +43,8 @@ required=(AI_HOSTNAME ADMIN_USER MODEL_DIR FAST_MODEL_FILE
   MEDIUM_MODEL_SHA256 LARGE_MODEL_FILE LARGE_MODEL_URL LARGE_MODEL_SHA256
   ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256
   EXPERIMENTAL_MODEL_FILE EXPERIMENTAL_MODEL_URL EXPERIMENTAL_MODEL_SHA256
-  VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256)
+  VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256
+  COMPRESSION_MODEL_FILE COMPRESSION_MODEL_URL COMPRESSION_MODEL_SHA256)
 for name in "${required[@]}"; do
   [[ -n "${!name:-}" ]] || die "Pflichtwert $name fehlt."
 done
@@ -339,6 +340,9 @@ UNCENSORED_MODEL_FILE=$UNCENSORED_MODEL_FILE
 UNCENSORED_PROJECTOR_FILE=$UNCENSORED_PROJECTOR_FILE
 EXPERIMENTAL_MODEL_FILE=$EXPERIMENTAL_MODEL_FILE
 VISION_PROJECTOR_FILE=$VISION_PROJECTOR_FILE
+COMPRESSION_MODEL_FILE=$COMPRESSION_MODEL_FILE
+COMPRESSION_CONTEXT=${COMPRESSION_CONTEXT:-65536}
+COMPRESSION_GPU_DEVICE=${COMPRESSION_GPU_DEVICE:-GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b}
 FAST_CONTEXT=${FAST_CONTEXT:-76800}
 MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-160000}
 LARGE_CONTEXT=${LARGE_CONTEXT:-192000}
@@ -400,6 +404,7 @@ $UNCENSORED_MODEL_FILE|$UNCENSORED_MODEL_URL|$UNCENSORED_MODEL_SHA256
 $UNCENSORED_PROJECTOR_FILE|$UNCENSORED_PROJECTOR_URL|$UNCENSORED_PROJECTOR_SHA256
 $EXPERIMENTAL_MODEL_FILE|$EXPERIMENTAL_MODEL_URL|$EXPERIMENTAL_MODEL_SHA256
 $VISION_PROJECTOR_FILE|$VISION_PROJECTOR_URL|$VISION_PROJECTOR_SHA256
+$COMPRESSION_MODEL_FILE|$COMPRESSION_MODEL_URL|$COMPRESSION_MODEL_SHA256
 EOF
 }
 
@@ -495,6 +500,8 @@ build_and_start() {
   docker compose --env-file "$SECRETS_DIR/stack.env" --profile image create flux-worker
   docker compose --env-file "$SECRETS_DIR/stack.env" up -d --build \
     xtts piper tts-gateway profile-controller router open-webui hermes
+  docker compose --env-file "$SECRETS_DIR/stack.env" --profile compression up -d \
+    llama-compression
 
   if [[ ${WIREGUARD_MODE:-container} == container ]]; then
     systemctl restart mike-ai-container-vpn-guard.service

platform/hermes/config.yaml

@@ -100,9 +100,15 @@ auxiliary:
   title_generation:
     enabled: false
   compression:
-    provider: "main"
+    provider: "openai-api"
-    model: ""
+    model: "qwen-compression"
+    base_url: "http://192.168.1.212:8099/v1"
+    api_key: "local"
     reasoning_effort: "none"
+    extra_body:
+      chat_template_kwargs:
+        enable_thinking: false
+    timeout: 600
 
 # The full hermes-cli preset injects several large, overlapping schemas on
 # every turn. Athena already exposes browsing, orchestration and host services

platform/hermes/install-profiles.sh

@@ -35,6 +35,14 @@ create_profile() {
   docker exec "$HERMES_CONTAINER" hermes -p "$name" config set compression.target_ratio 0.35
   docker exec "$HERMES_CONTAINER" hermes -p "$name" config set compression.protect_last_n 12
   docker exec "$HERMES_CONTAINER" hermes -p "$name" config set compression.protect_first_n 0
+  docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.provider openai-api
+  docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.model qwen-compression
+  docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.base_url http://192.168.1.212:8099/v1
+  docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.api_key local
+  docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.reasoning_effort none
+  docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.extra_body \
+    '{"chat_template_kwargs":{"enable_thinking":false}}'
+  docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.timeout 600
   docker exec "$HERMES_CONTAINER" hermes -p "$name" config set platform_toolsets.cli \
     '["web","terminal","file","skills","todo","memory","vision","tts"]'
   [[ $(docker exec "$HERMES_CONTAINER" hermes -p "$name" config get model.default) == "$model" ]] || \