diff --git a/.env.example b/.env.example index 486fcd2..ca3166f 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,7 @@ DEFAULT_REASONING_EFFORT=off FAST_MODEL_FILE=qwen-mix/Qwen3.8-27B-IQ4-MIX.gguf MEDIUM_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf +BETA1_MODEL_FILE=qwen3.8-27b-gsq-rco-test/Qwen3.8-27B-GSQ-RCO-IQ3_XXS-mtp.gguf LARGE_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf ULTRA_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf UNCENSORED_MODEL_FILE=qwen3.8-27b-abliterated/Qwen3.8-27B-ABLITERATED-Q4_K_M.gguf @@ -27,6 +28,9 @@ FAST_UBATCH_SIZE=32 MEDIUM_CONTEXT=160000 MEDIUM_BATCH_SIZE=2048 MEDIUM_UBATCH_SIZE=128 +BETA1_CONTEXT=192000 +BETA1_BATCH_SIZE=2048 +BETA1_UBATCH_SIZE=128 LARGE_CONTEXT=192000 LARGE_BATCH_SIZE=2048 LARGE_UBATCH_SIZE=128 @@ -39,6 +43,7 @@ UNCENSORED_UBATCH_SIZE=128 FAST_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b MEDIUM_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b MEDIUM_TENSOR_SPLIT=85,15 +BETA1_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b LARGE_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b LARGE_TENSOR_SPLIT=86,14 ULTRA_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b @@ -51,6 +56,7 @@ LLAMA_THREADS_BATCH=6 FAST_PARALLEL_SLOTS=1 LLAMA_CACHE_RAM_MIB=32768 MEDIUM_PARALLEL_SLOTS=1 +BETA1_PARALLEL_SLOTS=1 LARGE_PARALLEL_SLOTS=1 ULTRA_PARALLEL_SLOTS=1 UNCENSORED_PARALLEL_SLOTS=1 diff --git a/compose.yaml b/compose.yaml index 43490e5..e56a96b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -235,6 +235,88 @@ services: - --spec-draft-p-min - "0.05" + # Beta 1 keeps the complete GSQ-RCO text model and KV cache on the RTX 5080. + # Only the multimodal projector runs on the RTX 3060. The measured hard + # boundary is 196608 tokens; 192K deliberately retains runtime headroom. + llama-beta1: + <<: *llama-common + container_name: mike-ai-llama-beta1 + labels: + com.mike-ai.llama-profile: beta1 + environment: + NVIDIA_VISIBLE_DEVICES: ${BETA1_GPU_DEVICES:-0,1} + NVIDIA_DRIVER_CAPABILITIES: compute,utility + MTMD_BACKEND_DEVICE: CUDA1 + command: + - --model + - "/models/${BETA1_MODEL_FILE:?BETA1_MODEL_FILE is required}" + - --mmproj + - "/models/${VISION_PROJECTOR_FILE:?VISION_PROJECTOR_FILE is required}" + - --mmproj-offload + - --mmproj-device + - CUDA1 + - --alias + - qwen-beta-1 + - --ctx-size + - "${BETA1_CONTEXT:-192000}" + - --flash-attn + - "on" + - --cache-type-k + - q4_0 + - --cache-type-v + - q4_0 + - --cache-prompt + - --cache-ram + - "${LLAMA_CACHE_RAM_MIB:-32768}" + - --threads + - "${LLAMA_THREADS:-6}" + - --threads-batch + - "${LLAMA_THREADS_BATCH:-6}" + - --batch-size + - "${BETA1_BATCH_SIZE:-2048}" + - --ubatch-size + - "${BETA1_UBATCH_SIZE:-128}" + - --parallel + - "${BETA1_PARALLEL_SLOTS:-1}" + - --kv-unified + - --jinja + - --reasoning + - auto + - --reasoning-preserve + - --host + - 0.0.0.0 + - --port + - "8080" + - --metrics + - --fit + - "off" + - --n-gpu-layers + - all + - --no-mmap + - --no-ui + - --temperature + - "1.0" + - --top-p + - "0.95" + - --top-k + - "20" + - --device + - CUDA0 + - --main-gpu + - "0" + - --split-mode + - none + - --spec-type + - draft-mtp + - --spec-draft-n-max + - "3" + - --spec-draft-type-k + - f16 + - --spec-draft-type-v + - f16 + - --spec-draft-p-min + - "0.05" + llama-large: <<: *llama-common container_name: mike-ai-llama-large @@ -487,7 +569,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: CONTROLLER_TOKEN: "${CONTROLLER_TOKEN:?CONTROLLER_TOKEN is required}" - ALLOWED_PROFILES: fast,medium,large,ultra,uncensored + ALLOWED_PROFILES: fast,medium,beta1,large,ultra,uncensored IMAGE_WORKER: image networks: [control] security_opt: ["no-new-privileges:true"] diff --git a/config/install.env.example b/config/install.env.example index 58a6c91..42f3b54 100644 --- a/config/install.env.example +++ b/config/install.env.example @@ -56,6 +56,9 @@ FAST_MODEL_SHA256=54879ae8738d5938f46cb3b8cbf16bf42b8c85b7d68d7c73f062b612ec183e MEDIUM_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf MEDIUM_MODEL_URL=https://huggingface.co/jpetrina/Qwen3.8-27B-IQ4_XS-pure-GGUF/resolve/main/qwen3.8-27b-IQ4_XS-pure.gguf MEDIUM_MODEL_SHA256=ea5a3c45d407f9b9e5d2c0d647f0ea600f486f6b86b92b56d0823ba073dae675 +BETA1_MODEL_FILE=qwen3.8-27b-gsq-rco-test/Qwen3.8-27B-GSQ-RCO-IQ3_XXS-mtp.gguf +BETA1_MODEL_URL=https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/resolve/main/Qwen3.8-27B-GSQ-RCO-IQ3_XXS-mtp.gguf +BETA1_MODEL_SHA256=63f29a2189a6b4cc31f81e093d3856ad293a5583114439f41ae1ed7af4093262 LARGE_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf LARGE_MODEL_URL=https://huggingface.co/jpetrina/Qwen3.8-27B-IQ4_XS-pure-GGUF/resolve/main/qwen3.8-27b-IQ4_XS-pure.gguf LARGE_MODEL_SHA256=ea5a3c45d407f9b9e5d2c0d647f0ea600f486f6b86b92b56d0823ba073dae675 @@ -82,6 +85,11 @@ MEDIUM_CONTEXT=160000 MEDIUM_BATCH_SIZE=2048 MEDIUM_UBATCH_SIZE=128 MEDIUM_TENSOR_SPLIT=85,15 +BETA1_CONTEXT=192000 +BETA1_BATCH_SIZE=2048 +BETA1_UBATCH_SIZE=128 +BETA1_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b +BETA1_PARALLEL_SLOTS=1 LARGE_CONTEXT=192000 LARGE_BATCH_SIZE=2048 LARGE_UBATCH_SIZE=128 diff --git a/config/profile-matrix.json b/config/profile-matrix.json index 27f5572..0f5cf62 100644 --- a/config/profile-matrix.json +++ b/config/profile-matrix.json @@ -27,6 +27,18 @@ "mtp": 3, "description": "Ausgewogenes Standardprofil für Alltag und lange agentische Aufgaben." }, + { + "id": "beta1", + "alias": "qwen-beta-1", + "context": 192000, + "parallel_slots": 1, + "model_env": "BETA1_MODEL_FILE", + "model_family": "Qwen3.8-27B GSQ-RCO IQ3_XXS MTP", + "gpu_split": "5080 model / 3060 vision", + "vision": true, + "mtp": 3, + "description": "Beta 1: schnelles GSQ-RCO-Testprofil mit 192K Kontext und Vision-Projektor auf der RTX 3060." + }, { "id": "large", "alias": "qwen-large", diff --git a/docs/GSQ_RCO_BETA1_20260904.md b/docs/GSQ_RCO_BETA1_20260904.md new file mode 100644 index 0000000..c8b3031 --- /dev/null +++ b/docs/GSQ_RCO_BETA1_20260904.md @@ -0,0 +1,32 @@ +# Qwen Beta 1 – GSQ-RCO + +`qwen-beta-1` ist ein zusätzliches, nicht standardmäßig aktives Router-Profil. +Die bestehenden Profile und das Standardprofil `qwen-medium` bleiben unverändert. + +## Laufzeitkonfiguration + +- Modell: Qwen3.8-27B GSQ-RCO IQ3_XXS MTP +- Kontext: 192.000 Token +- Textmodell und KV-Cache: vollständig RTX 5080 +- Vision-Projektor: RTX 3060 +- KV-Quantisierung: Q4_0 für K und V +- MTP: 3 Draft-Token +- Batch / Micro-Batch: 2048 / 128 + +## Gemessene Kontextgrenze + +Eine echte Bildanfrage mit einer 2,3-MB-JPEG-Datei wurde zur Bestimmung der +VRAM-Grenze verwendet. + +| Kontext | Ergebnis | Rest auf RTX 5080 nach Bildlauf | +|---:|---|---:| +| 192.000 | bestanden | ca. 129 MiB | +| 196.608 | bestanden, harte Kante | ca. 9 MiB | +| 197.120 | CUDA Out of Memory | ca. 1 MiB vor Abbruch | + +Der produktive Beta-Modus verwendet deshalb 192.000 Token. 196.608 ist nur +die gemessene technische Obergrenze und besitzt keine ausreichende Reserve +für einen verlässlichen Dauerbetrieb. + +Beim erfolgreichen 196.608-Test erreichte die Bildanfrage rund 366 Prompt- +Token/s und 85 Ausgabe-Token/s. Das erkannte Bild wurde korrekt beschrieben. diff --git a/docs/STANDARD_PROFILE_MATRIX.md b/docs/STANDARD_PROFILE_MATRIX.md index 5a9537d..b2e3ea4 100644 --- a/docs/STANDARD_PROFILE_MATRIX.md +++ b/docs/STANDARD_PROFILE_MATRIX.md @@ -8,6 +8,7 @@ Standardprofil: **medium** · globales Ausgabelimit: **8192 Token** |---|---|---:|---:|---|---|---|---:| | fast | `qwen-fast` | 76,800 | 1 | Qwen3.8-27B IQ4 Mix | 5080 only | ja | 2 | | medium | `qwen-medium` | 160,000 | 1 | Qwen3.8-27B IQ4 XS Pure | 85:15 | ja | 3 | +| beta1 | `qwen-beta-1` | 192,000 | 1 | Qwen3.8-27B GSQ-RCO IQ3_XXS MTP | 5080 model / 3060 vision | ja | 3 | | large | `qwen-large` | 192,000 | 1 | Qwen3.8-27B IQ4 XS Pure | 86:14 | ja | 3 | | ultra | `qwen-ultra` | 262,144 | 1 | Qwen3.8-27B IQ4 XS Pure | 80:20 | nein | 2 | | uncensored | `qwen-uncensored` | 80,000 | 1 | Qwen3.8-27B Abliterated Q4_K_M | 90:10 | ja | 2 | @@ -16,6 +17,7 @@ Standardprofil: **medium** · globales Ausgabelimit: **8192 Token** - **fast**: Schnelles Profil für kurze Chats und zügige Werkzeugaufgaben. - **medium**: Ausgewogenes Standardprofil für Alltag und lange agentische Aufgaben. +- **beta1**: Beta 1: schnelles GSQ-RCO-Testprofil mit 192K Kontext und Vision-Projektor auf der RTX 3060. - **large**: Großes Profil für umfangreiche Dokumente und lange technische Arbeiten. - **ultra**: Maximaler Textkontext; bewusst ohne Vision-Projektor. - **uncensored**: Weniger restriktives Spezialprofil; Werkzeugrechte bleiben unverändert. diff --git a/install.sh b/install.sh index 53f6214..b9a356f 100755 --- a/install.sh +++ b/install.sh @@ -41,6 +41,7 @@ source "$CONFIG" required=(AI_HOSTNAME ADMIN_USER MODEL_DIR FAST_MODEL_FILE FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL MEDIUM_MODEL_SHA256 LARGE_MODEL_FILE LARGE_MODEL_URL LARGE_MODEL_SHA256 + BETA1_MODEL_FILE BETA1_MODEL_URL BETA1_MODEL_SHA256 ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256 UNCENSORED_MODEL_FILE UNCENSORED_MODEL_URL UNCENSORED_MODEL_SHA256 UNCENSORED_PROJECTOR_FILE UNCENSORED_PROJECTOR_URL @@ -319,6 +320,7 @@ XTTS_GPU_DEVICE=${XTTS_GPU_DEVICE:-GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b} AI_DNS=${WG_DNS:-1.1.1.1} FAST_MODEL_FILE=$FAST_MODEL_FILE MEDIUM_MODEL_FILE=$MEDIUM_MODEL_FILE +BETA1_MODEL_FILE=$BETA1_MODEL_FILE LARGE_MODEL_FILE=$LARGE_MODEL_FILE ULTRA_MODEL_FILE=$ULTRA_MODEL_FILE UNCENSORED_MODEL_FILE=$UNCENSORED_MODEL_FILE @@ -332,6 +334,10 @@ MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-160000} MEDIUM_BATCH_SIZE=${MEDIUM_BATCH_SIZE:-2048} MEDIUM_UBATCH_SIZE=${MEDIUM_UBATCH_SIZE:-128} MEDIUM_PARALLEL_SLOTS=${MEDIUM_PARALLEL_SLOTS:-1} +BETA1_CONTEXT=${BETA1_CONTEXT:-192000} +BETA1_BATCH_SIZE=${BETA1_BATCH_SIZE:-2048} +BETA1_UBATCH_SIZE=${BETA1_UBATCH_SIZE:-128} +BETA1_PARALLEL_SLOTS=${BETA1_PARALLEL_SLOTS:-1} LARGE_CONTEXT=${LARGE_CONTEXT:-192000} LARGE_BATCH_SIZE=${LARGE_BATCH_SIZE:-2048} LARGE_UBATCH_SIZE=${LARGE_UBATCH_SIZE:-128} @@ -347,6 +353,7 @@ UNCENSORED_PARALLEL_SLOTS=${UNCENSORED_PARALLEL_SLOTS:-1} FAST_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES} MEDIUM_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES} MEDIUM_TENSOR_SPLIT=${MEDIUM_TENSOR_SPLIT:-90,10} +BETA1_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES} LARGE_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES} LARGE_TENSOR_SPLIT=${LARGE_TENSOR_SPLIT:-86,14} ULTRA_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES} @@ -394,6 +401,7 @@ download_models() { done </dev/null 2>&1 || docker volume create portainer_data >/dev/null docker compose --env-file "$SECRETS_DIR/stack.env" stop llama-dashboard portainer diff --git a/platform/scripts/llama-profile b/platform/scripts/llama-profile index 8859dfd..a5b64d0 100755 --- a/platform/scripts/llama-profile +++ b/platform/scripts/llama-profile @@ -7,9 +7,9 @@ SERVICE="${LLAMA_SERVICE:-mike-ai-llama-ui.service}" PROFILE="${1:-}" case "$PROFILE" in - fast|medium|large|ultra|uncensored) ;; + fast|medium|beta1|large|ultra|uncensored) ;; *) - echo "Usage: llama-profile {fast|medium|large|ultra|uncensored}" >&2 + echo "Usage: llama-profile {fast|medium|beta1|large|ultra|uncensored}" >&2 exit 2 ;; esac diff --git a/router/ai_profile_router.py b/router/ai_profile_router.py index 4c20357..adf6bf4 100755 --- a/router/ai_profile_router.py +++ b/router/ai_profile_router.py @@ -2,20 +2,22 @@ """AI Profile Router – OpenAI-kompatibler Proxy vor llama.cpp. Leitet OpenAI-kompatible Requests transparent an den lokalen llama.cpp-Server -weiter (Streaming, Tool Calls, JSON) und schaltet zwischen fünf festen +weiter (Streaming, Tool Calls, JSON) und schaltet zwischen sechs festen Profilen um: Profil Kontext ------ -------- fast 76800 medium 160000 + beta1 192000 large 192000 ultra 262144 uncensored 80000 -Virtuelle Modelle: qwen-fast, qwen-medium, qwen-large, qwen-ultra, - qwen-uncensored -Kommandos: POST /fast, /medium, /large, /ultra, /uncensored +Virtuelle Modelle: qwen-fast, qwen-medium, qwen-beta-1, qwen-large, + qwen-ultra, qwen-uncensored +Kommandos: POST /fast, /medium, /beta1, /large, /ultra, + /uncensored GET /status (Zustand) Bildgenerierung und Editing (FLUX.2-klein-4B): @@ -207,7 +209,10 @@ PROFILES = {name: definition["context"] for name, definition in PROFILE_REGISTRY.items()} EXPECTED_MODELS = {name: definition.get("model_alias") for name, definition in PROFILE_REGISTRY.items()} -VIRTUAL_MODELS = {f"qwen-{name}": name for name in PROFILES} +VIRTUAL_MODELS = { + (EXPECTED_MODELS.get(name) or f"qwen-{name}"): name + for name in PROFILES +} log = logging.getLogger("ai-profile-router") AUTH: AuthPolicy | None = None @@ -1524,11 +1529,11 @@ class Handler(BaseHTTPRequestHandler): self._images_list() elif path.startswith("/images/") and self.command == "GET": self._image_serve(path[len("/images/"):]) - elif (path in ("/fast", "/medium", "/large", "/ultra", "/uncensored") + elif (path.removeprefix("/") in PROFILES and (self.command == "POST" or (self.command == "GET" and ALLOW_LEGACY_GET_SWITCH))): self._switch(path[1:]) - elif (path in ("/fast", "/medium", "/large", "/ultra", "/uncensored") + elif (path.removeprefix("/") in PROFILES and self.command == "GET"): self._send_error(405, "Profilwechsel erfordert POST", "invalid_request_error", "method_not_allowed") @@ -1673,7 +1678,7 @@ class Handler(BaseHTTPRequestHandler): def _models_payload() -> dict: models = [ { - "id": f"qwen-{name}", + "id": EXPECTED_MODELS.get(name) or f"qwen-{name}", "object": "model", "created": 0, "owned_by": "ai-profile-router", diff --git a/router/router_profiles.json b/router/router_profiles.json index ceb40c0..4782b65 100644 --- a/router/router_profiles.json +++ b/router/router_profiles.json @@ -8,6 +8,10 @@ "context": 160000, "model_alias": "qwen-medium" }, + "beta1": { + "context": 192000, + "model_alias": "qwen-beta-1" + }, "large": { "context": 192000, "model_alias": "qwen-large" diff --git a/smoke-test.sh b/smoke-test.sh index a3b211a..aeb60ff 100755 --- a/smoke-test.sh +++ b/smoke-test.sh @@ -38,7 +38,7 @@ done pass "Athena-Kerndienste sind gesund" active_llama=$(docker ps --format '{{.Names}}' | \ - grep -Ec '^mike-ai-llama-(fast|medium|large|ultra|uncensored)$' || true) + grep -Ec '^mike-ai-llama-(fast|medium|beta1|large|ultra|uncensored)$' || true) [[ $active_llama -eq 1 ]] || fail "$active_llama aktive llama-Profile (erwartet: 1)" pass "Genau ein llama.cpp-Profil ist aktiv"