diff --git a/.env.example b/.env.example index 38f57d1..ad06c9a 100644 --- a/.env.example +++ b/.env.example @@ -19,7 +19,6 @@ LARGE_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf ULTRA_MODEL_FILE=qwen-pure/qwen3.8-27b-IQ4_XS-pure.gguf UNCENSORED_MODEL_FILE=qwen3.8-27b-abliterated/Qwen3.8-27B-ABLITERATED-Q4_K_M.gguf UNCENSORED_PROJECTOR_FILE=qwen3.8-27b-abliterated/mmproj-Qwen3.8-27B-ABLITERATED-F16.gguf -EXPERIMENTAL_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf FAST_CONTEXT=76800 @@ -37,7 +36,6 @@ ULTRA_UBATCH_SIZE=128 UNCENSORED_CONTEXT=80000 UNCENSORED_BATCH_SIZE=2048 UNCENSORED_UBATCH_SIZE=128 -EXPERIMENTAL_CONTEXT=76800 FAST_GPU_DEVICES=0,1 MEDIUM_GPU_DEVICES=0,1 MEDIUM_TENSOR_SPLIT=85,15 @@ -48,7 +46,6 @@ ULTRA_TENSOR_SPLIT=80,20 UNCENSORED_GPU_DEVICES=0,1 UNCENSORED_TENSOR_SPLIT=90,10 UNCENSORED_MTP_MAX=2 -EXPERIMENTAL_GPU_DEVICES=0 LLAMA_THREADS=6 LLAMA_THREADS_BATCH=6 FAST_PARALLEL_SLOTS=1 @@ -57,4 +54,3 @@ MEDIUM_PARALLEL_SLOTS=1 LARGE_PARALLEL_SLOTS=1 ULTRA_PARALLEL_SLOTS=1 UNCENSORED_PARALLEL_SLOTS=1 -EXPERIMENTAL_PARALLEL_SLOTS=1 diff --git a/compose.yaml b/compose.yaml index 13e05c5..0d3b48c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -471,55 +471,6 @@ services: - --spec-draft-type-v - f16 - llama-experimental: - <<: *llama-common - container_name: mike-ai-llama-experimental - labels: - com.mike-ai.llama-profile: experimental - environment: - NVIDIA_VISIBLE_DEVICES: ${EXPERIMENTAL_GPU_DEVICES:-0} - NVIDIA_DRIVER_CAPABILITIES: compute,utility - command: - - --model - - "/models/${EXPERIMENTAL_MODEL_FILE:?EXPERIMENTAL_MODEL_FILE is required}" - - --alias - - qwen-experimental - - --ctx-size - - "${EXPERIMENTAL_CONTEXT:-76800}" - - --flash-attn - - "on" - - --cache-type-k - - q4_0 - - --cache-type-v - - q4_0 - - --cache-prompt - - --cache-reuse - - "${LLAMA_CACHE_REUSE:-256}" - - --cache-ram - - "${LLAMA_CACHE_RAM_MIB:-24576}" - - --parallel - - "${EXPERIMENTAL_PARALLEL_SLOTS:-1}" - - --kv-unified - - --jinja - - --reasoning - - auto - - --reasoning-preserve - - --host - - 0.0.0.0 - - --port - - "8080" - - --metrics - - --fit - - "off" - - --n-gpu-layers - - all - - --no-mmap - - --no-ui - - --device - - CUDA0 - - --split-mode - - none - profile-controller: build: ./platform/docker/profile-controller image: mike-ai/profile-controller:local @@ -531,7 +482,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: CONTROLLER_TOKEN: "${CONTROLLER_TOKEN:?CONTROLLER_TOKEN is required}" - ALLOWED_PROFILES: fast,medium,large,ultra,uncensored,experimental + ALLOWED_PROFILES: fast,medium,large,ultra,uncensored IMAGE_WORKER: image networks: [control] security_opt: ["no-new-privileges:true"] diff --git a/config/install.env.example b/config/install.env.example index 7cf4e89..6b07847 100644 --- a/config/install.env.example +++ b/config/install.env.example @@ -65,9 +65,6 @@ UNCENSORED_MODEL_SHA256=5d53637a59cfcd3a4d8354e254ffd44943e5a693da2405a3e228c629 UNCENSORED_PROJECTOR_FILE=qwen3.8-27b-abliterated/mmproj-Qwen3.8-27B-ABLITERATED-F16.gguf UNCENSORED_PROJECTOR_URL=https://huggingface.co/Blackfrost-AI/Qwen3.8-27B-ABLITERATED-GGUF/resolve/main/mmproj-Qwen3.8-27B-ABLITERATED-F16.gguf UNCENSORED_PROJECTOR_SHA256=2284099ce864f1023d721e6ef5eaef32bb56abdbc1dc561c6d91300f12ef2e4b -EXPERIMENTAL_MODEL_FILE=qwen/Qwen3.8-27B-UD-IQ4_XS.gguf -EXPERIMENTAL_MODEL_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-UD-IQ4_XS.gguf -EXPERIMENTAL_MODEL_SHA256=40fac4050e940397dbf13087afd50f4734a11805bf9d65ef8ddd7483470e6199 VISION_PROJECTOR_FILE=qwen/mmproj-BF16.gguf VISION_PROJECTOR_URL=https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/mmproj-BF16.gguf VISION_PROJECTOR_SHA256=83ee4f4f205fa514161778c41df1ea14144faa0f713510893b63c2395f5c2d53 @@ -95,7 +92,6 @@ UNCENSORED_BATCH_SIZE=2048 UNCENSORED_UBATCH_SIZE=128 UNCENSORED_TENSOR_SPLIT=90,10 UNCENSORED_MTP_MAX=2 -EXPERIMENTAL_CONTEXT=76800 LLAMA_THREADS=6 LLAMA_THREADS_BATCH=6 FAST_PARALLEL_SLOTS=1 @@ -104,7 +100,6 @@ MEDIUM_PARALLEL_SLOTS=1 LARGE_PARALLEL_SLOTS=1 ULTRA_PARALLEL_SLOTS=1 UNCENSORED_PARALLEL_SLOTS=1 -EXPERIMENTAL_PARALLEL_SLOTS=1 PIPER_TTS_VERSION=1.6.0 PIPER_VOICE=de_DE-thorsten-high XTTS_IMAGE=ghcr.io/coqui-ai/xtts-streaming-server:latest-cuda121@sha256:f7fb3b1f9d4bc88af94da1b5959d8002f1e0b003c97557164034eb8a29f01b90 diff --git a/config/profile-matrix.json b/config/profile-matrix.json index 98e89bb..27f5572 100644 --- a/config/profile-matrix.json +++ b/config/profile-matrix.json @@ -19,7 +19,7 @@ "id": "medium", "alias": "qwen-medium", "context": 160000, - "parallel_slots": 2, + "parallel_slots": 1, "model_env": "MEDIUM_MODEL_FILE", "model_family": "Qwen3.8-27B IQ4 XS Pure", "gpu_split": "85:15", diff --git a/docs/STANDARD_PROFILE_MATRIX.md b/docs/STANDARD_PROFILE_MATRIX.md index 5ffdfac..5a9537d 100644 --- a/docs/STANDARD_PROFILE_MATRIX.md +++ b/docs/STANDARD_PROFILE_MATRIX.md @@ -7,7 +7,7 @@ Standardprofil: **medium** ยท globales Ausgabelimit: **8192 Token** | Profil | API-Alias | Gesamtkontext | Slots | Modell | GPU-Verteilung | Vision | MTP | |---|---|---:|---:|---|---|---|---:| | fast | `qwen-fast` | 76,800 | 1 | Qwen3.8-27B IQ4 Mix | 5080 only | ja | 2 | -| medium | `qwen-medium` | 160,000 | 2 | Qwen3.8-27B IQ4 XS Pure | 85:15 | ja | 3 | +| medium | `qwen-medium` | 160,000 | 1 | Qwen3.8-27B IQ4 XS Pure | 85:15 | ja | 3 | | large | `qwen-large` | 192,000 | 1 | Qwen3.8-27B IQ4 XS Pure | 86:14 | ja | 3 | | ultra | `qwen-ultra` | 262,144 | 1 | Qwen3.8-27B IQ4 XS Pure | 80:20 | nein | 2 | | uncensored | `qwen-uncensored` | 80,000 | 1 | Qwen3.8-27B Abliterated Q4_K_M | 90:10 | ja | 2 | diff --git a/install.sh b/install.sh index 1363f8c..919bc0b 100755 --- a/install.sh +++ b/install.sh @@ -42,7 +42,6 @@ required=(AI_HOSTNAME ADMIN_USER MODEL_DIR FAST_MODEL_FILE FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL MEDIUM_MODEL_SHA256 LARGE_MODEL_FILE LARGE_MODEL_URL LARGE_MODEL_SHA256 ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256 - EXPERIMENTAL_MODEL_FILE EXPERIMENTAL_MODEL_URL EXPERIMENTAL_MODEL_SHA256 VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256) for name in "${required[@]}"; do [[ -n "${!name:-}" ]] || die "Pflichtwert $name fehlt." @@ -321,14 +320,12 @@ LARGE_MODEL_FILE=$LARGE_MODEL_FILE ULTRA_MODEL_FILE=$ULTRA_MODEL_FILE UNCENSORED_MODEL_FILE=$UNCENSORED_MODEL_FILE UNCENSORED_PROJECTOR_FILE=$UNCENSORED_PROJECTOR_FILE -EXPERIMENTAL_MODEL_FILE=$EXPERIMENTAL_MODEL_FILE VISION_PROJECTOR_FILE=$VISION_PROJECTOR_FILE FAST_CONTEXT=${FAST_CONTEXT:-76800} MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-160000} LARGE_CONTEXT=${LARGE_CONTEXT:-192000} ULTRA_CONTEXT=${ULTRA_CONTEXT:-262144} UNCENSORED_CONTEXT=${UNCENSORED_CONTEXT:-80000} -EXPERIMENTAL_CONTEXT=${EXPERIMENTAL_CONTEXT:-76800} FAST_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES} MEDIUM_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES} MEDIUM_TENSOR_SPLIT=${MEDIUM_TENSOR_SPLIT:-90,10} @@ -339,7 +336,6 @@ ULTRA_TENSOR_SPLIT=${ULTRA_TENSOR_SPLIT:-80,20} UNCENSORED_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES} UNCENSORED_TENSOR_SPLIT=${UNCENSORED_TENSOR_SPLIT:-90,10} UNCENSORED_MTP_MAX=${UNCENSORED_MTP_MAX:-2} -EXPERIMENTAL_GPU_DEVICES=${TEXT_GPU_DEVICES:-0} IMAGE_GPU_DEVICES=${IMAGE_GPU_DEVICES:-${TEXT_GPU_DEVICES:-0}} FLUX_MODEL_DIR=${FLUX_MODEL_DIR:-/data/models/FLUX.2-klein-4B} LLAMA_THREADS=${LLAMA_THREADS:-6} @@ -382,7 +378,6 @@ $LARGE_MODEL_FILE|$LARGE_MODEL_URL|$LARGE_MODEL_SHA256 $ULTRA_MODEL_FILE|$ULTRA_MODEL_URL|$ULTRA_MODEL_SHA256 $UNCENSORED_MODEL_FILE|$UNCENSORED_MODEL_URL|$UNCENSORED_MODEL_SHA256 $UNCENSORED_PROJECTOR_FILE|$UNCENSORED_PROJECTOR_URL|$UNCENSORED_PROJECTOR_SHA256 -$EXPERIMENTAL_MODEL_FILE|$EXPERIMENTAL_MODEL_URL|$EXPERIMENTAL_MODEL_SHA256 $VISION_PROJECTOR_FILE|$VISION_PROJECTOR_URL|$VISION_PROJECTOR_SHA256 EOF }