Add Qwen GSQ-RCO Beta 1 profile

2026-09-04 14:36:50 +02:00
parent 744a207e5a
commit f553108912
11 changed files with 172 additions and 13 deletions
@@ -15,6 +15,7 @@ DEFAULT_REASONING_EFFORT=off

 FAST_MODEL_FILE=qwen-mix/Qwen3.8-27B-IQ4-MIX.gguf
 MEDIUM_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf
+BETA1_MODEL_FILE=qwen3.8-27b-gsq-rco-test/Qwen3.8-27B-GSQ-RCO-IQ3_XXS-mtp.gguf
 LARGE_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf
 ULTRA_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf
 UNCENSORED_MODEL_FILE=qwen3.8-27b-abliterated/Qwen3.8-27B-ABLITERATED-Q4_K_M.gguf
@@ -27,6 +28,9 @@ FAST_UBATCH_SIZE=32
 MEDIUM_CONTEXT=160000
 MEDIUM_BATCH_SIZE=2048
 MEDIUM_UBATCH_SIZE=128
+BETA1_CONTEXT=192000
+BETA1_BATCH_SIZE=2048
+BETA1_UBATCH_SIZE=128
 LARGE_CONTEXT=192000
 LARGE_BATCH_SIZE=2048
 LARGE_UBATCH_SIZE=128
@@ -39,6 +43,7 @@ UNCENSORED_UBATCH_SIZE=128
 FAST_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
 MEDIUM_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
 MEDIUM_TENSOR_SPLIT=85,15
+BETA1_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
 LARGE_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
 LARGE_TENSOR_SPLIT=86,14
 ULTRA_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
@@ -51,6 +56,7 @@ LLAMA_THREADS_BATCH=6
 FAST_PARALLEL_SLOTS=1
 LLAMA_CACHE_RAM_MIB=32768
 MEDIUM_PARALLEL_SLOTS=1
+BETA1_PARALLEL_SLOTS=1
 LARGE_PARALLEL_SLOTS=1
 ULTRA_PARALLEL_SLOTS=1
 UNCENSORED_PARALLEL_SLOTS=1
@@ -235,6 +235,88 @@ services:
      - --spec-draft-p-min
      - "0.05"

+  # Beta 1 keeps the complete GSQ-RCO text model and KV cache on the RTX 5080.
+  # Only the multimodal projector runs on the RTX 3060. The measured hard
+  # boundary is 196608 tokens; 192K deliberately retains runtime headroom.
+  llama-beta1:
+    <<: *llama-common
+    container_name: mike-ai-llama-beta1
+    labels:
+      com.mike-ai.llama-profile: beta1
+    environment:
+      NVIDIA_VISIBLE_DEVICES: ${BETA1_GPU_DEVICES:-0,1}
+      NVIDIA_DRIVER_CAPABILITIES: compute,utility
+      MTMD_BACKEND_DEVICE: CUDA1
+    command:
+      - --model
+      - "/models/${BETA1_MODEL_FILE:?BETA1_MODEL_FILE is required}"
+      - --mmproj
+      - "/models/${VISION_PROJECTOR_FILE:?VISION_PROJECTOR_FILE is required}"
+      - --mmproj-offload
+      - --mmproj-device
+      - CUDA1
+      - --alias
+      - qwen-beta-1
+      - --ctx-size
+      - "${BETA1_CONTEXT:-192000}"
+      - --flash-attn
+      - "on"
+      - --cache-type-k
+      - q4_0
+      - --cache-type-v
+      - q4_0
+      - --cache-prompt
+      - --cache-ram
+      - "${LLAMA_CACHE_RAM_MIB:-32768}"
+      - --threads
+      - "${LLAMA_THREADS:-6}"
+      - --threads-batch
+      - "${LLAMA_THREADS_BATCH:-6}"
+      - --batch-size
+      - "${BETA1_BATCH_SIZE:-2048}"
+      - --ubatch-size
+      - "${BETA1_UBATCH_SIZE:-128}"
+      - --parallel
+      - "${BETA1_PARALLEL_SLOTS:-1}"
+      - --kv-unified
+      - --jinja
+      - --reasoning
+      - auto
+      - --reasoning-preserve
+      - --host
+      - 0.0.0.0
+      - --port
+      - "8080"
+      - --metrics
+      - --fit
+      - "off"
+      - --n-gpu-layers
+      - all
+      - --no-mmap
+      - --no-ui
+      - --temperature
+      - "1.0"
+      - --top-p
+      - "0.95"
+      - --top-k
+      - "20"
+      - --device
+      - CUDA0
+      - --main-gpu
+      - "0"
+      - --split-mode
+      - none
+      - --spec-type
+      - draft-mtp
+      - --spec-draft-n-max
+      - "3"
+      - --spec-draft-type-k
+      - f16
+      - --spec-draft-type-v
+      - f16
+      - --spec-draft-p-min
+      - "0.05"
+
  llama-large:
    <<: *llama-common
    container_name: mike-ai-llama-large
@@ -487,7 +569,7 @@ services:
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      CONTROLLER_TOKEN: "${CONTROLLER_TOKEN:?CONTROLLER_TOKEN is required}"
-      ALLOWED_PROFILES: fast,medium,large,ultra,uncensored
+      ALLOWED_PROFILES: fast,medium,beta1,large,ultra,uncensored
      IMAGE_WORKER: image
    networks: [control]
    security_opt: ["no-new-privileges:true"]
@@ -56,6 +56,9 @@ FAST_MODEL_SHA256=54879ae8738d5938f46cb3b8cbf16bf42b8c85b7d68d7c73f062b612ec183e
 MEDIUM_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf
 MEDIUM_MODEL_URL=https://huggingface.co/jpetrina/Qwen3.8-27B-IQ4_XS-pure-GGUF/resolve/main/qwen3.8-27b-IQ4_XS-pure.gguf
 MEDIUM_MODEL_SHA256=ea5a3c45d407f9b9e5d2c0d647f0ea600f486f6b86b92b56d0823ba073dae675
+BETA1_MODEL_FILE=qwen3.8-27b-gsq-rco-test/Qwen3.8-27B-GSQ-RCO-IQ3_XXS-mtp.gguf
+BETA1_MODEL_URL=https://huggingface.co/ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF/resolve/main/Qwen3.8-27B-GSQ-RCO-IQ3_XXS-mtp.gguf
+BETA1_MODEL_SHA256=63f29a2189a6b4cc31f81e093d3856ad293a5583114439f41ae1ed7af4093262
 LARGE_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf
 LARGE_MODEL_URL=https://huggingface.co/jpetrina/Qwen3.8-27B-IQ4_XS-pure-GGUF/resolve/main/qwen3.8-27b-IQ4_XS-pure.gguf
 LARGE_MODEL_SHA256=ea5a3c45d407f9b9e5d2c0d647f0ea600f486f6b86b92b56d0823ba073dae675
@@ -82,6 +85,11 @@ MEDIUM_CONTEXT=160000
 MEDIUM_BATCH_SIZE=2048
 MEDIUM_UBATCH_SIZE=128
 MEDIUM_TENSOR_SPLIT=85,15
+BETA1_CONTEXT=192000
+BETA1_BATCH_SIZE=2048
+BETA1_UBATCH_SIZE=128
+BETA1_GPU_DEVICES=GPU-8ad38c6c-5a01-9d8e-1dfa-ed662ad78fbe,GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b
+BETA1_PARALLEL_SLOTS=1
 LARGE_CONTEXT=192000
 LARGE_BATCH_SIZE=2048
 LARGE_UBATCH_SIZE=128
@@ -27,6 +27,18 @@
      "mtp": 3,
      "description": "Ausgewogenes Standardprofil für Alltag und lange agentische Aufgaben."
    },
+    {
+      "id": "beta1",
+      "alias": "qwen-beta-1",
+      "context": 192000,
+      "parallel_slots": 1,
+      "model_env": "BETA1_MODEL_FILE",
+      "model_family": "Qwen3.8-27B GSQ-RCO IQ3_XXS MTP",
+      "gpu_split": "5080 model / 3060 vision",
+      "vision": true,
+      "mtp": 3,
+      "description": "Beta 1: schnelles GSQ-RCO-Testprofil mit 192K Kontext und Vision-Projektor auf der RTX 3060."
+    },
    {
      "id": "large",
      "alias": "qwen-large",
@@ -0,0 +1,32 @@
+# Qwen Beta 1 – GSQ-RCO
+
+`qwen-beta-1` ist ein zusätzliches, nicht standardmäßig aktives Router-Profil.
+Die bestehenden Profile und das Standardprofil `qwen-medium` bleiben unverändert.
+
+## Laufzeitkonfiguration
+
+- Modell: Qwen3.8-27B GSQ-RCO IQ3_XXS MTP
+- Kontext: 192.000 Token
+- Textmodell und KV-Cache: vollständig RTX 5080
+- Vision-Projektor: RTX 3060
+- KV-Quantisierung: Q4_0 für K und V
+- MTP: 3 Draft-Token
+- Batch / Micro-Batch: 2048 / 128
+
+## Gemessene Kontextgrenze
+
+Eine echte Bildanfrage mit einer 2,3-MB-JPEG-Datei wurde zur Bestimmung der
+VRAM-Grenze verwendet.
+
+| Kontext | Ergebnis | Rest auf RTX 5080 nach Bildlauf |
+|---:|---|---:|
+| 192.000 | bestanden | ca. 129 MiB |
+| 196.608 | bestanden, harte Kante | ca. 9 MiB |
+| 197.120 | CUDA Out of Memory | ca. 1 MiB vor Abbruch |
+
+Der produktive Beta-Modus verwendet deshalb 192.000 Token. 196.608 ist nur
+die gemessene technische Obergrenze und besitzt keine ausreichende Reserve
+für einen verlässlichen Dauerbetrieb.
+
+Beim erfolgreichen 196.608-Test erreichte die Bildanfrage rund 366 Prompt-
+Token/s und 85 Ausgabe-Token/s. Das erkannte Bild wurde korrekt beschrieben.
@@ -8,6 +8,7 @@ Standardprofil: **medium** · globales Ausgabelimit: **8192 Token**
 |---|---|---:|---:|---|---|---|---:|
 | fast | `qwen-fast` | 76,800 | 1 | Qwen3.8-27B IQ4 Mix | 5080 only | ja | 2 |
 | medium | `qwen-medium` | 160,000 | 1 | Qwen3.8-27B IQ4 XS Pure | 85:15 | ja | 3 |
+| beta1 | `qwen-beta-1` | 192,000 | 1 | Qwen3.8-27B GSQ-RCO IQ3_XXS MTP | 5080 model / 3060 vision | ja | 3 |
 | large | `qwen-large` | 192,000 | 1 | Qwen3.8-27B IQ4 XS Pure | 86:14 | ja | 3 |
 | ultra | `qwen-ultra` | 262,144 | 1 | Qwen3.8-27B IQ4 XS Pure | 80:20 | nein | 2 |
 | uncensored | `qwen-uncensored` | 80,000 | 1 | Qwen3.8-27B Abliterated Q4_K_M | 90:10 | ja | 2 |
@@ -16,6 +17,7 @@ Standardprofil: **medium** · globales Ausgabelimit: **8192 Token**

 - **fast**: Schnelles Profil für kurze Chats und zügige Werkzeugaufgaben.
 - **medium**: Ausgewogenes Standardprofil für Alltag und lange agentische Aufgaben.
+- **beta1**: Beta 1: schnelles GSQ-RCO-Testprofil mit 192K Kontext und Vision-Projektor auf der RTX 3060.
 - **large**: Großes Profil für umfangreiche Dokumente und lange technische Arbeiten.
 - **ultra**: Maximaler Textkontext; bewusst ohne Vision-Projektor.
 - **uncensored**: Weniger restriktives Spezialprofil; Werkzeugrechte bleiben unverändert.
@@ -41,6 +41,7 @@ source "$CONFIG"
 required=(AI_HOSTNAME ADMIN_USER MODEL_DIR FAST_MODEL_FILE
  FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL
  MEDIUM_MODEL_SHA256 LARGE_MODEL_FILE LARGE_MODEL_URL LARGE_MODEL_SHA256
+  BETA1_MODEL_FILE BETA1_MODEL_URL BETA1_MODEL_SHA256
  ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256
  UNCENSORED_MODEL_FILE UNCENSORED_MODEL_URL UNCENSORED_MODEL_SHA256
  UNCENSORED_PROJECTOR_FILE UNCENSORED_PROJECTOR_URL
@@ -319,6 +320,7 @@ XTTS_GPU_DEVICE=${XTTS_GPU_DEVICE:-GPU-4834d9d7-5b61-3004-1fb3-4ae49d482d4b}
 AI_DNS=${WG_DNS:-1.1.1.1}
 FAST_MODEL_FILE=$FAST_MODEL_FILE
 MEDIUM_MODEL_FILE=$MEDIUM_MODEL_FILE
+BETA1_MODEL_FILE=$BETA1_MODEL_FILE
 LARGE_MODEL_FILE=$LARGE_MODEL_FILE
 ULTRA_MODEL_FILE=$ULTRA_MODEL_FILE
 UNCENSORED_MODEL_FILE=$UNCENSORED_MODEL_FILE
@@ -332,6 +334,10 @@ MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-160000}
 MEDIUM_BATCH_SIZE=${MEDIUM_BATCH_SIZE:-2048}
 MEDIUM_UBATCH_SIZE=${MEDIUM_UBATCH_SIZE:-128}
 MEDIUM_PARALLEL_SLOTS=${MEDIUM_PARALLEL_SLOTS:-1}
+BETA1_CONTEXT=${BETA1_CONTEXT:-192000}
+BETA1_BATCH_SIZE=${BETA1_BATCH_SIZE:-2048}
+BETA1_UBATCH_SIZE=${BETA1_UBATCH_SIZE:-128}
+BETA1_PARALLEL_SLOTS=${BETA1_PARALLEL_SLOTS:-1}
 LARGE_CONTEXT=${LARGE_CONTEXT:-192000}
 LARGE_BATCH_SIZE=${LARGE_BATCH_SIZE:-2048}
 LARGE_UBATCH_SIZE=${LARGE_UBATCH_SIZE:-128}
@@ -347,6 +353,7 @@ UNCENSORED_PARALLEL_SLOTS=${UNCENSORED_PARALLEL_SLOTS:-1}
 FAST_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
 MEDIUM_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
 MEDIUM_TENSOR_SPLIT=${MEDIUM_TENSOR_SPLIT:-90,10}
+BETA1_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
 LARGE_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
 LARGE_TENSOR_SPLIT=${LARGE_TENSOR_SPLIT:-86,14}
 ULTRA_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
@@ -394,6 +401,7 @@ download_models() {
  done <<EOF
 $FAST_MODEL_FILE|$FAST_MODEL_URL|$FAST_MODEL_SHA256
 $MEDIUM_MODEL_FILE|$MEDIUM_MODEL_URL|$MEDIUM_MODEL_SHA256
+$BETA1_MODEL_FILE|$BETA1_MODEL_URL|$BETA1_MODEL_SHA256
 $LARGE_MODEL_FILE|$LARGE_MODEL_URL|$LARGE_MODEL_SHA256
 $ULTRA_MODEL_FILE|$ULTRA_MODEL_URL|$ULTRA_MODEL_SHA256
 $UNCENSORED_MODEL_FILE|$UNCENSORED_MODEL_URL|$UNCENSORED_MODEL_SHA256
@@ -488,7 +496,7 @@ build_and_start() {
  # Portable MCPs and Hermes live on Unraid and are restored through Appdata.
  "$STACK_DIR/platform/mcp/install-tools.sh"
  docker compose --env-file "$SECRETS_DIR/stack.env" --profile inference create \
-    llama-fast llama-medium llama-large llama-ultra llama-uncensored
+    llama-fast llama-medium llama-beta1 llama-large llama-ultra llama-uncensored
  docker compose --env-file "$SECRETS_DIR/stack.env" --profile image create image-worker
  docker volume inspect portainer_data >/dev/null 2>&1 || docker volume create portainer_data >/dev/null
  docker compose --env-file "$SECRETS_DIR/stack.env" stop llama-dashboard portainer
@@ -7,9 +7,9 @@ SERVICE="${LLAMA_SERVICE:-mike-ai-llama-ui.service}"
 PROFILE="${1:-}"

 case "$PROFILE" in
-  fast|medium|large|ultra|uncensored) ;;
+  fast|medium|beta1|large|ultra|uncensored) ;;
  *)
-    echo "Usage: llama-profile {fast|medium|large|ultra|uncensored}" >&2
+    echo "Usage: llama-profile {fast|medium|beta1|large|ultra|uncensored}" >&2
    exit 2
    ;;
 esac
@@ -2,20 +2,22 @@
 """AI Profile Router – OpenAI-kompatibler Proxy vor llama.cpp.

 Leitet OpenAI-kompatible Requests transparent an den lokalen llama.cpp-Server
-weiter (Streaming, Tool Calls, JSON) und schaltet zwischen fünf festen
+weiter (Streaming, Tool Calls, JSON) und schaltet zwischen sechs festen
 Profilen um:

    Profil   Kontext
    ------   --------
    fast     76800
    medium   160000
+    beta1    192000
    large    192000
    ultra    262144
    uncensored 80000

-Virtuelle Modelle:  qwen-fast, qwen-medium, qwen-large, qwen-ultra,
-                    qwen-uncensored
-Kommandos:          POST /fast, /medium, /large, /ultra, /uncensored
+Virtuelle Modelle:  qwen-fast, qwen-medium, qwen-beta-1, qwen-large,
+                    qwen-ultra, qwen-uncensored
+Kommandos:          POST /fast, /medium, /beta1, /large, /ultra,
+                    /uncensored
                    GET  /status                 (Zustand)

 Bildgenerierung und Editing (FLUX.2-klein-4B):
@@ -207,7 +209,10 @@ PROFILES = {name: definition["context"]
            for name, definition in PROFILE_REGISTRY.items()}
 EXPECTED_MODELS = {name: definition.get("model_alias")
                   for name, definition in PROFILE_REGISTRY.items()}
-VIRTUAL_MODELS = {f"qwen-{name}": name for name in PROFILES}
+VIRTUAL_MODELS = {
+    (EXPECTED_MODELS.get(name) or f"qwen-{name}"): name
+    for name in PROFILES
+}

 log = logging.getLogger("ai-profile-router")
 AUTH: AuthPolicy | None = None
@@ -1524,11 +1529,11 @@ class Handler(BaseHTTPRequestHandler):
                self._images_list()
            elif path.startswith("/images/") and self.command == "GET":
                self._image_serve(path[len("/images/"):])
-            elif (path in ("/fast", "/medium", "/large", "/ultra", "/uncensored")
+            elif (path.removeprefix("/") in PROFILES
                  and (self.command == "POST"
                       or (self.command == "GET" and ALLOW_LEGACY_GET_SWITCH))):
                self._switch(path[1:])
-            elif (path in ("/fast", "/medium", "/large", "/ultra", "/uncensored")
+            elif (path.removeprefix("/") in PROFILES
                  and self.command == "GET"):
                self._send_error(405, "Profilwechsel erfordert POST",
                                 "invalid_request_error", "method_not_allowed")
@@ -1673,7 +1678,7 @@ class Handler(BaseHTTPRequestHandler):
    def _models_payload() -> dict:
        models = [
            {
-                "id": f"qwen-{name}",
+                "id": EXPECTED_MODELS.get(name) or f"qwen-{name}",
                "object": "model",
                "created": 0,
                "owned_by": "ai-profile-router",
@@ -8,6 +8,10 @@
      "context": 160000,
      "model_alias": "qwen-medium"
    },
+    "beta1": {
+      "context": 192000,
+      "model_alias": "qwen-beta-1"
+    },
    "large": {
      "context": 192000,
      "model_alias": "qwen-large"
@@ -38,7 +38,7 @@ done
 pass "Athena-Kerndienste sind gesund"

 active_llama=$(docker ps --format '{{.Names}}' | \
-  grep -Ec '^mike-ai-llama-(fast|medium|large|ultra|uncensored)$' || true)
+  grep -Ec '^mike-ai-llama-(fast|medium|beta1|large|ultra|uncensored)$' || true)
 [[ $active_llama -eq 1 ]] || fail "$active_llama aktive llama-Profile (erwartet: 1)"
 pass "Genau ein llama.cpp-Profil ist aktiv"