From f52cf14069b174459ee43944e0b0e339d95c9edb Mon Sep 17 00:00:00 2001 From: Mikei386 <44135113+Mikei386@users.noreply.github.com> Date: Sat, 22 Aug 2026 10:44:14 +0200 Subject: [PATCH] Add tested 256K Ultra profile --- .env.example | 4 + README.md | 6 +- compose.yaml | 73 ++++++++++++++++++- config/install.env.example | 5 ++ docs/ARCHITECTURE.md | 1 + docs/OPERATIONS.md | 7 +- install.sh | 8 +- platform/checks/verify-platform.sh | 2 +- .../profile-controller/profile_controller.py | 2 +- router/ai_profile_router.py | 11 +-- router/router_profiles.json | 4 + router/router_support.py | 1 + 12 files changed, 111 insertions(+), 13 deletions(-) diff --git a/.env.example b/.env.example index fb90a26..b0b35fd 100644 --- a/.env.example +++ b/.env.example @@ -12,16 +12,20 @@ AI_DNS=192.168.1.1 FAST_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf MEDIUM_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf LONG_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf +ULTRA_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf EXPERIMENTAL_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf FAST_CONTEXT=76800 MEDIUM_CONTEXT=94208 LONG_CONTEXT=131072 +ULTRA_CONTEXT=262144 EXPERIMENTAL_CONTEXT=76800 FAST_GPU_DEVICES=0 MEDIUM_GPU_DEVICES=0 LONG_GPU_DEVICES=0 +ULTRA_GPU_DEVICES=0,1 +ULTRA_TENSOR_SPLIT=80,20 EXPERIMENTAL_GPU_DEVICES=0 LLAMA_THREADS=6 LLAMA_THREADS_BATCH=6 diff --git a/README.md b/README.md index bf27476..8325eed 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,10 @@ WireGuard-Isolation. - Debian 13 als schlanker GPU-Host - llama.cpp selbst gebaut und auf einen geprüften Commit festgelegt -- vier getrennte Profilcontainer, davon immer exakt einer aktiv -- `/fast`, `/medium`, `/long` und `/experimental` über den Profile Router +- vier schaltbare Profilcontainer plus ein isolierter Experimentalcontainer; + davon ist immer exakt ein Inferenzcontainer aktiv +- `/fast`, `/medium`, `/long` und `/ultra` über den Profile Router +- `/ultra`: getestetes text-only 256K-Profil (IQ4-MIX, beide GPUs, 80:20); etwa 59 Token/s im Referenzlauf und erfolgreicher 220K-Prompt-Fülltest - Open WebUI als einzige normale Oberfläche - SearXNG/Web-MCP ohne externen API-Schlüssel - zentrale MCP-Werkzeugebene: getrennte Container für Web, HA, ARR, Unraid diff --git a/compose.yaml b/compose.yaml index dc9381e..755a2eb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -222,6 +222,77 @@ services: - --spec-draft-type-v - q4_0 + # Text-only long-context profile. This exact IQ4-MIX / 256K / 80:20 + # combination completed the 220K fill test on RTX 5080 + RTX 3060. + # Deliberately no vision projector: Ultra prioritizes maximum usable context. + llama-ultra: + <<: *llama-common + container_name: mike-ai-llama-ultra + labels: + com.mike-ai.llama-profile: ultra + environment: + NVIDIA_VISIBLE_DEVICES: ${ULTRA_GPU_DEVICES:-0,1} + NVIDIA_DRIVER_CAPABILITIES: compute,utility + command: + - --model + - "/models/${ULTRA_MODEL_FILE:?ULTRA_MODEL_FILE is required}" + - --alias + - qwen-ultra + - --ctx-size + - "${ULTRA_CONTEXT:-262144}" + - --flash-attn + - "on" + - --cache-type-k + - q4_0 + - --cache-type-v + - q4_0 + - --threads + - "${LLAMA_THREADS:-6}" + - --threads-batch + - "${LLAMA_THREADS_BATCH:-6}" + - --batch-size + - "64" + - --ubatch-size + - "32" + - --parallel + - "1" + - --jinja + - --reasoning + - auto + - --host + - 0.0.0.0 + - --port + - "8080" + - --metrics + - --fit + - "off" + - --n-gpu-layers + - all + - --no-mmap + - --no-ui + - --temperature + - "0.2" + - --top-p + - "0.8" + - --top-k + - "20" + - --device + - CUDA0,CUDA1 + - --main-gpu + - "0" + - --split-mode + - layer + - --tensor-split + - "${ULTRA_TENSOR_SPLIT:-80,20}" + - --spec-type + - draft-mtp + - --spec-draft-n-max + - "2" + - --spec-draft-type-k + - f16 + - --spec-draft-type-v + - f16 + llama-experimental: <<: *llama-common container_name: mike-ai-llama-experimental @@ -276,7 +347,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: CONTROLLER_TOKEN: "${CONTROLLER_TOKEN:?CONTROLLER_TOKEN is required}" - ALLOWED_PROFILES: fast,medium,long,experimental + ALLOWED_PROFILES: fast,medium,long,ultra,experimental networks: [control] security_opt: ["no-new-privileges:true"] healthcheck: diff --git a/config/install.env.example b/config/install.env.example index 36048fd..b198214 100644 --- a/config/install.env.example +++ b/config/install.env.example @@ -37,6 +37,9 @@ MEDIUM_MODEL_SHA256=40fac4050e940397dbf13087afd50f4734a11805bf9d65ef8ddd7483470e LONG_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf LONG_MODEL_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-UD-IQ4_XS.gguf LONG_MODEL_SHA256=40fac4050e940397dbf13087afd50f4734a11805bf9d65ef8ddd7483470e6199 +ULTRA_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf +ULTRA_MODEL_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-UD-IQ4_XS.gguf +ULTRA_MODEL_SHA256=40fac4050e940397dbf13087afd50f4734a11805bf9d65ef8ddd7483470e6199 EXPERIMENTAL_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf EXPERIMENTAL_MODEL_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-UD-IQ4_XS.gguf EXPERIMENTAL_MODEL_SHA256=40fac4050e940397dbf13087afd50f4734a11805bf9d65ef8ddd7483470e6199 @@ -49,6 +52,8 @@ VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c239 FAST_CONTEXT=76800 MEDIUM_CONTEXT=94208 LONG_CONTEXT=131072 +ULTRA_CONTEXT=262144 +ULTRA_TENSOR_SPLIT=80,20 EXPERIMENTAL_CONTEXT=76800 LLAMA_THREADS=6 LLAMA_THREADS_BATCH=6 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5bd9e9c..627f5f6 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -22,6 +22,7 @@ Heimnetz / VPN-Clients +-- llama-medium > exakt einer aktiv +-- llama-long --/ +-- llama-experimental + +-- llama-ultra (256K, text-only, dual GPU) +-- Piper-TTS (CPU, nur intern) +-- internes MCP-Netz +-- Web-MCP + TinySearch + SearXNG diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 3d681fb..447f8e1 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -74,9 +74,12 @@ Community Store nachgeladen. | Fast | `qwen-fast` | 76.800 | Alltag, Agenten, hohe Geschwindigkeit, integrierte Vision | | Medium | `qwen-medium` | 94.208 | mehr Kontext, reine IQ4_XS-Variante | | Long | `qwen-long` | 131.072 | lange Hermes-/MCP-Sitzungen | +| Ultra | `qwen-ultra` | 262.144 | maximaler Textkontext, IQ4-MIX auf RTX 5080 + RTX 3060 (80:20), ohne Vision-Projektor | -Manuell wird mit `llama-profile fast|medium|long` gewechselt. Über HTTP stehen -`POST /fast`, `/medium` und `/long` zur Verfügung. Für eine spätere Version ist +Manuell wird mit `llama-profile fast|medium|long|ultra` gewechselt. Über HTTP stehen +`POST /fast`, `/medium`, `/long` und `/ultra` zur Verfügung. Ultra erreichte im +Referenzlauf etwa 59 Token/s; ein Prompt-Fülltest mit rund 220.000 Tokens war +erfolgreich. Für eine spätere Version ist `large` als Alias für `long` vorgesehen; bestehende Namen bleiben kompatibel. ## Clients diff --git a/install.sh b/install.sh index 8a6ed90..303500a 100755 --- a/install.sh +++ b/install.sh @@ -41,6 +41,7 @@ source "$CONFIG" required=(AI_HOSTNAME ADMIN_USER AI_BIND_ADDRESS MODEL_DIR FAST_MODEL_FILE FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL MEDIUM_MODEL_SHA256 LONG_MODEL_FILE LONG_MODEL_URL LONG_MODEL_SHA256 + ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256 EXPERIMENTAL_MODEL_FILE EXPERIMENTAL_MODEL_URL EXPERIMENTAL_MODEL_SHA256 VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256) for name in "${required[@]}"; do @@ -209,15 +210,19 @@ AI_DNS=${WG_DNS:-1.1.1.1} FAST_MODEL_FILE=$FAST_MODEL_FILE MEDIUM_MODEL_FILE=$MEDIUM_MODEL_FILE LONG_MODEL_FILE=$LONG_MODEL_FILE +ULTRA_MODEL_FILE=$ULTRA_MODEL_FILE EXPERIMENTAL_MODEL_FILE=$EXPERIMENTAL_MODEL_FILE VISION_PROJECTOR_FILE=$VISION_PROJECTOR_FILE FAST_CONTEXT=${FAST_CONTEXT:-76800} MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-94208} LONG_CONTEXT=${LONG_CONTEXT:-131072} +ULTRA_CONTEXT=${ULTRA_CONTEXT:-262144} EXPERIMENTAL_CONTEXT=${EXPERIMENTAL_CONTEXT:-76800} FAST_GPU_DEVICES=${TEXT_GPU_DEVICES:-0} MEDIUM_GPU_DEVICES=${TEXT_GPU_DEVICES:-0} LONG_GPU_DEVICES=${TEXT_GPU_DEVICES:-0} +ULTRA_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES} +ULTRA_TENSOR_SPLIT=${ULTRA_TENSOR_SPLIT:-80,20} EXPERIMENTAL_GPU_DEVICES=${TEXT_GPU_DEVICES:-0} LLAMA_THREADS=${LLAMA_THREADS:-6} LLAMA_THREADS_BATCH=${LLAMA_THREADS_BATCH:-6} @@ -256,6 +261,7 @@ download_models() { $FAST_MODEL_FILE|$FAST_MODEL_URL|$FAST_MODEL_SHA256 $MEDIUM_MODEL_FILE|$MEDIUM_MODEL_URL|$MEDIUM_MODEL_SHA256 $LONG_MODEL_FILE|$LONG_MODEL_URL|$LONG_MODEL_SHA256 +$ULTRA_MODEL_FILE|$ULTRA_MODEL_URL|$ULTRA_MODEL_SHA256 $EXPERIMENTAL_MODEL_FILE|$EXPERIMENTAL_MODEL_URL|$EXPERIMENTAL_MODEL_SHA256 $VISION_PROJECTOR_FILE|$VISION_PROJECTOR_URL|$VISION_PROJECTOR_SHA256 EOF @@ -323,7 +329,7 @@ build_and_start() { # secret files and required local artifacts are present. "$STACK_DIR/platform/mcp/install-tools.sh" docker compose --env-file "$SECRETS_DIR/stack.env" --profile inference create \ - llama-fast llama-medium llama-long llama-experimental + llama-fast llama-medium llama-long llama-ultra llama-experimental docker compose --env-file "$SECRETS_DIR/stack.env" up -d --build \ profile-controller router open-webui diff --git a/platform/checks/verify-platform.sh b/platform/checks/verify-platform.sh index 5493574..9436740 100755 --- a/platform/checks/verify-platform.sh +++ b/platform/checks/verify-platform.sh @@ -40,7 +40,7 @@ for container in mike-ai-profile-controller mike-ai-router mike-ai-piper mike-ai fi done -ACTIVE_LLAMA="$(docker ps --format '{{.Names}}' | grep -Ec '^mike-ai-llama-(fast|medium|long|experimental)$' || true)" +ACTIVE_LLAMA="$(docker ps --format '{{.Names}}' | grep -Ec '^mike-ai-llama-(fast|medium|long|ultra|experimental)$' || true)" if [[ $ACTIVE_LLAMA -eq 1 ]]; then pass "exakt ein llama.cpp-Profil aktiv" else diff --git a/platform/docker/profile-controller/profile_controller.py b/platform/docker/profile-controller/profile_controller.py index 177ae7e..a83fa4a 100644 --- a/platform/docker/profile-controller/profile_controller.py +++ b/platform/docker/profile-controller/profile_controller.py @@ -21,7 +21,7 @@ PORT = int(os.environ.get("CONTROLLER_PORT", "8090")) SOCKET_PATH = os.environ.get("DOCKER_SOCKET", "/var/run/docker.sock") TOKEN_FILE = os.environ.get("CONTROLLER_TOKEN_FILE", "/run/secrets/controller-token") ALLOWED = tuple(x.strip() for x in os.environ.get( - "ALLOWED_PROFILES", "fast,medium,long,experimental").split(",") if x.strip()) + "ALLOWED_PROFILES", "fast,medium,long,ultra,experimental").split(",") if x.strip()) LABEL_KEY = "com.mike-ai.llama-profile" LOCK = threading.Lock() log = logging.getLogger("profile-controller") diff --git a/router/ai_profile_router.py b/router/ai_profile_router.py index cd1d51b..0b7dcf4 100755 --- a/router/ai_profile_router.py +++ b/router/ai_profile_router.py @@ -2,7 +2,7 @@ """AI Profile Router – OpenAI-kompatibler Proxy vor llama.cpp. Leitet OpenAI-kompatible Requests transparent an den lokalen llama.cpp-Server -weiter (Streaming, Tool Calls, JSON) und schaltet zwischen drei festen +weiter (Streaming, Tool Calls, JSON) und schaltet zwischen vier festen Profilen um: Profil Kontext @@ -10,9 +10,10 @@ Profilen um: fast 76800 medium 94208 long 131072 + ultra 262144 -Virtuelle Modelle: qwen-fast, qwen-medium, qwen-long -Kommandos: POST /fast, /medium, /long (Profilwechsel) +Virtuelle Modelle: qwen-fast, qwen-medium, qwen-long, qwen-ultra +Kommandos: POST /fast, /medium, /long, /ultra (Profilwechsel) GET /status (Zustand) Bildgenerierung (FLUX.2 [klein] 4B Base): @@ -1109,11 +1110,11 @@ class Handler(BaseHTTPRequestHandler): self._images_list() elif path.startswith("/images/") and self.command == "GET": self._image_serve(path[len("/images/"):]) - elif (path in ("/fast", "/medium", "/long") + elif (path in ("/fast", "/medium", "/long", "/ultra") and (self.command == "POST" or (self.command == "GET" and ALLOW_LEGACY_GET_SWITCH))): self._switch(path[1:]) - elif (path in ("/fast", "/medium", "/long") + elif (path in ("/fast", "/medium", "/long", "/ultra") and self.command == "GET"): self._send_error(405, "Profilwechsel erfordert POST", "invalid_request_error", "method_not_allowed") diff --git a/router/router_profiles.json b/router/router_profiles.json index 1a95481..f486e62 100644 --- a/router/router_profiles.json +++ b/router/router_profiles.json @@ -11,6 +11,10 @@ "long": { "context": 131072, "model_alias": "qwen-long" + }, + "ultra": { + "context": 262144, + "model_alias": "qwen-ultra" } } } diff --git a/router/router_support.py b/router/router_support.py index 7ed70cd..c4715d8 100644 --- a/router/router_support.py +++ b/router/router_support.py @@ -36,6 +36,7 @@ def load_profile_registry(path: str | None) -> dict[str, dict]: "fast": {"context": 76800, "model_alias": None}, "medium": {"context": 94208, "model_alias": None}, "long": {"context": 131072, "model_alias": None}, + "ultra": {"context": 262144, "model_alias": None}, } if not path: return fallback