diff --git a/.env.example b/.env.example index d9ffd75..ee68dc9 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,11 @@ XTTS_CACHE_DIR=/data/models/xtts-v2-cache XTTS_GPU_DEVICE=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b AI_DNS=192.168.1.1 +# Dedicated Hermes context compressor on the RTX 3060. +COMPRESSION_MODEL_FILE=qwen3.5-2b-compression/Qwen_Qwen3.5-2B-Q4_K_M.gguf +COMPRESSION_CONTEXT=65536 +COMPRESSION_GPU_DEVICE=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b + FAST_MODEL_FILE=qwen-mix/Qwen3.8-27B-IQ4-MIX.gguf MEDIUM_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf LARGE_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf diff --git a/README.md b/README.md index 3052022..e49a335 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,9 @@ zusammen mit dem übrigen Appdata gesichert. - `compose.yaml` ist der einzige Einstieg für die KI-Dienste auf Athena. - Genau ein llama.cpp-Profil ist aktiv. Der Router schaltet zwischen Fast, Medium, Large, Ultra und Uncensored. +- Ein unabhängiges Qwen3.5-2B-Modell auf der RTX 3060 komprimiert Hermes-Chats + über `http://192.168.1.212:8099/v1`. Es läuft ohne Thinking mit 65K Kontext + und blockiert dadurch weder Router noch das aktive 27B-Hauptmodell. - Der **Athena Operator** bleibt als einziger hostgebundener administrativer MCP direkt auf Athena. MCPHub veröffentlicht seinen vorhandenen WireGuard-HTTP-Endpunkt zentral unter `/mcp/athena-operator`; es gibt keinen diff --git a/compose.yaml b/compose.yaml index c782763..44152e5 100644 --- a/compose.yaml +++ b/compose.yaml @@ -65,6 +65,75 @@ services: retries: 12 start_period: 10s + # Small, non-thinking side model used only for Hermes context compression. + # It shares the WireGuard namespace, so Unraid can reach port 8099 while the + # university LAN receives no published host port. The stable GPU UUID keeps + # it on the RTX 3060 even if PCI enumeration changes again. + llama-compression: + image: ${LLAMA_IMAGE:-mike-ai/llama.cpp:local} + container_name: mike-ai-llama-compression + restart: unless-stopped + profiles: [compression] + gpus: all + network_mode: "service:wireguard-gateway" + depends_on: + wireguard-gateway: + condition: service_healthy + read_only: true + tmpfs: + - /tmp:size=256m,mode=1777 + volumes: + - "${MODEL_DIR:-/srv/mike-ai/models}:/models:ro" + environment: + NVIDIA_VISIBLE_DEVICES: ${COMPRESSION_GPU_DEVICE:-GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b} + NVIDIA_DRIVER_CAPABILITIES: compute,utility + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + command: + - --model + - "/models/${COMPRESSION_MODEL_FILE:-qwen3.5-2b-compression/Qwen_Qwen3.5-2B-Q4_K_M.gguf}" + - --alias + - qwen-compression + - --ctx-size + - "${COMPRESSION_CONTEXT:-65536}" + - --flash-attn + - "on" + - --cache-type-k + - q4_0 + - --cache-type-v + - q4_0 + - --cache-prompt + - --parallel + - "1" + - --jinja + - --host + - 0.0.0.0 + - --port + - "8099" + - --metrics + - --n-gpu-layers + - all + - --device + - CUDA0 + - --split-mode + - none + - --no-mmap + - --no-ui + - --batch-size + - "512" + - --ubatch-size + - "256" + - --threads + - "${LLAMA_THREADS:-6}" + - --threads-batch + - "${LLAMA_THREADS_BATCH:-6}" + healthcheck: + test: [CMD, curl, -fsS, "http://127.0.0.1:8099/health"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 30s + llama-fast: <<: *llama-common container_name: mike-ai-llama-fast diff --git a/config/install.env.example b/config/install.env.example index 71ae05b..f5d0b23 100644 --- a/config/install.env.example +++ b/config/install.env.example @@ -72,6 +72,11 @@ EXPERIMENTAL_MODEL_SHA256=40fac4050e940397dbf13087afd50f4734a11805bf9d65ef8ddd74 VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf VISION_PROJECTOR_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/mmproj-BF16.gguf VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c2395f5c2d53 +COMPRESSION_MODEL_FILE=qwen3.5-2b-compression/Qwen_Qwen3.5-2B-Q4_K_M.gguf +COMPRESSION_MODEL_URL=https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF/resolve/main/Qwen_Qwen3.5-2B-Q4_K_M.gguf +COMPRESSION_MODEL_SHA256=57a1085840f497d764a7fc5d346922dbde961efb54cc792ea81d694fd846a1d8 +COMPRESSION_CONTEXT=65536 +COMPRESSION_GPU_DEVICE=GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b # All standard profiles use the MTP tensor embedded in their GGUF. A separate # draft-model artifact is neither downloaded nor passed to llama-server. diff --git a/docs/STANDARD_PROFILE_MATRIX.md b/docs/STANDARD_PROFILE_MATRIX.md index 94552a3..91eb142 100644 --- a/docs/STANDARD_PROFILE_MATRIX.md +++ b/docs/STANDARD_PROFILE_MATRIX.md @@ -30,6 +30,20 @@ und einer bewussten Aktualisierung dieser Datei. - Das gesonderte Experimentalprofil gehört nicht zur Benutzer-Matrix und wird in Open WebUI nicht als reguläres Modell angeboten. +## Hermes-Kompressionsmodell + +Das Hilfsmodell `qwen-compression` ist kein Chatprofil. Qwen3.5-2B Q4_K_M +läuft unabhängig mit 65.536 Token Kontext auf der RTX 3060 und ist nur über +WireGuard auf Port 8099 erreichbar. Hermes erzwingt dafür Non-Thinking. + +- 30.245 Token, kalter Prompt: 4.519,7 Prefill-Tok/s. +- Ausgabe im 30K-Qualitätstest: 117,1 Tok/s; vier von fünf ungefilterten + Ankern bewahrt. +- Echter Hermes-Hilfsmodellaufruf: 158 Token in 4,6 Sekunden; alle geforderten + Pfade, Ports, Verbote und der Commit-Anker wurden bewahrt. +- Zusätzlicher Speicherbedarf auf der RTX 3060: rund 1,8 GiB; nach dem Laden + blieben im gemessenen Produktionszustand rund 4,5 GiB frei. + ## Nachweise - Fast 76,8K: synthetischer Referenzlauf, 85,50 Tok/s. diff --git a/install.sh b/install.sh index 4173442..3c6c920 100755 --- a/install.sh +++ b/install.sh @@ -43,7 +43,8 @@ required=(AI_HOSTNAME ADMIN_USER MODEL_DIR FAST_MODEL_FILE MEDIUM_MODEL_SHA256 LARGE_MODEL_FILE LARGE_MODEL_URL LARGE_MODEL_SHA256 ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256 EXPERIMENTAL_MODEL_FILE EXPERIMENTAL_MODEL_URL EXPERIMENTAL_MODEL_SHA256 - VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256) + VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256 + COMPRESSION_MODEL_FILE COMPRESSION_MODEL_URL COMPRESSION_MODEL_SHA256) for name in "${required[@]}"; do [[ -n "${!name:-}" ]] || die "Pflichtwert $name fehlt." done @@ -339,6 +340,9 @@ UNCENSORED_MODEL_FILE=$UNCENSORED_MODEL_FILE UNCENSORED_PROJECTOR_FILE=$UNCENSORED_PROJECTOR_FILE EXPERIMENTAL_MODEL_FILE=$EXPERIMENTAL_MODEL_FILE VISION_PROJECTOR_FILE=$VISION_PROJECTOR_FILE +COMPRESSION_MODEL_FILE=$COMPRESSION_MODEL_FILE +COMPRESSION_CONTEXT=${COMPRESSION_CONTEXT:-65536} +COMPRESSION_GPU_DEVICE=${COMPRESSION_GPU_DEVICE:-GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b} FAST_CONTEXT=${FAST_CONTEXT:-76800} MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-160000} LARGE_CONTEXT=${LARGE_CONTEXT:-192000} @@ -400,6 +404,7 @@ $UNCENSORED_MODEL_FILE|$UNCENSORED_MODEL_URL|$UNCENSORED_MODEL_SHA256 $UNCENSORED_PROJECTOR_FILE|$UNCENSORED_PROJECTOR_URL|$UNCENSORED_PROJECTOR_SHA256 $EXPERIMENTAL_MODEL_FILE|$EXPERIMENTAL_MODEL_URL|$EXPERIMENTAL_MODEL_SHA256 $VISION_PROJECTOR_FILE|$VISION_PROJECTOR_URL|$VISION_PROJECTOR_SHA256 +$COMPRESSION_MODEL_FILE|$COMPRESSION_MODEL_URL|$COMPRESSION_MODEL_SHA256 EOF } @@ -495,6 +500,8 @@ build_and_start() { docker compose --env-file "$SECRETS_DIR/stack.env" --profile image create flux-worker docker compose --env-file "$SECRETS_DIR/stack.env" up -d --build \ xtts piper tts-gateway profile-controller router open-webui hermes + docker compose --env-file "$SECRETS_DIR/stack.env" --profile compression up -d \ + llama-compression if [[ ${WIREGUARD_MODE:-container} == container ]]; then systemctl restart mike-ai-container-vpn-guard.service diff --git a/platform/hermes/config.yaml b/platform/hermes/config.yaml index d4981d8..70401da 100644 --- a/platform/hermes/config.yaml +++ b/platform/hermes/config.yaml @@ -100,9 +100,15 @@ auxiliary: title_generation: enabled: false compression: - provider: "main" - model: "" + provider: "openai-api" + model: "qwen-compression" + base_url: "http://192.168.1.212:8099/v1" + api_key: "local" reasoning_effort: "none" + extra_body: + chat_template_kwargs: + enable_thinking: false + timeout: 600 # The full hermes-cli preset injects several large, overlapping schemas on # every turn. Athena already exposes browsing, orchestration and host services diff --git a/platform/hermes/install-profiles.sh b/platform/hermes/install-profiles.sh index 71f4f64..cfc4b66 100755 --- a/platform/hermes/install-profiles.sh +++ b/platform/hermes/install-profiles.sh @@ -35,6 +35,14 @@ create_profile() { docker exec "$HERMES_CONTAINER" hermes -p "$name" config set compression.target_ratio 0.35 docker exec "$HERMES_CONTAINER" hermes -p "$name" config set compression.protect_last_n 12 docker exec "$HERMES_CONTAINER" hermes -p "$name" config set compression.protect_first_n 0 + docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.provider openai-api + docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.model qwen-compression + docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.base_url http://192.168.1.212:8099/v1 + docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.api_key local + docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.reasoning_effort none + docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.extra_body \ + '{"chat_template_kwargs":{"enable_thinking":false}}' + docker exec "$HERMES_CONTAINER" hermes -p "$name" config set auxiliary.compression.timeout 600 docker exec "$HERMES_CONTAINER" hermes -p "$name" config set platform_toolsets.cli \ '["web","terminal","file","skills","todo","memory","vision","tts"]' [[ $(docker exec "$HERMES_CONTAINER" hermes -p "$name" config get model.default) == "$model" ]] || \