michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Define four-profile production matrix with Medium default

Browse Source
This commit is contained in:
Mikei386
2026-08-22 11:30:38 +02:00
parent 977f8f5c76
commit 41b9c17f0f
33 changed files with 237 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
install.sh
+15 -13
View File
@@ -40,7 +40,7 @@ source "$CONFIG"
required=(AI_HOSTNAME ADMIN_USER AI_BIND_ADDRESS MODEL_DIR FAST_MODEL_FILE
FAST_MODEL_URL FAST_MODEL_SHA256 MEDIUM_MODEL_FILE MEDIUM_MODEL_URL
MEDIUM_MODEL_SHA256 LONG_MODEL_FILE LONG_MODEL_URL LONG_MODEL_SHA256
MEDIUM_MODEL_SHA256 LARGE_MODEL_FILE LARGE_MODEL_URL LARGE_MODEL_SHA256
ULTRA_MODEL_FILE ULTRA_MODEL_URL ULTRA_MODEL_SHA256
EXPERIMENTAL_MODEL_FILE EXPERIMENTAL_MODEL_URL EXPERIMENTAL_MODEL_SHA256
VISION_PROJECTOR_FILE VISION_PROJECTOR_URL VISION_PROJECTOR_SHA256)
@@ -209,18 +209,20 @@ PIPER_VOICE=${PIPER_VOICE:-de_DE-thorsten-high}
AI_DNS=${WG_DNS:-1.1.1.1}
FAST_MODEL_FILE=$FAST_MODEL_FILE
MEDIUM_MODEL_FILE=$MEDIUM_MODEL_FILE
LONG_MODEL_FILE=$LONG_MODEL_FILE
LARGE_MODEL_FILE=$LARGE_MODEL_FILE
ULTRA_MODEL_FILE=$ULTRA_MODEL_FILE
EXPERIMENTAL_MODEL_FILE=$EXPERIMENTAL_MODEL_FILE
VISION_PROJECTOR_FILE=$VISION_PROJECTOR_FILE
FAST_CONTEXT=${FAST_CONTEXT:-76800}
MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-94208}
LONG_CONTEXT=${LONG_CONTEXT:-131072}
MEDIUM_CONTEXT=${MEDIUM_CONTEXT:-160000}
LARGE_CONTEXT=${LARGE_CONTEXT:-192000}
ULTRA_CONTEXT=${ULTRA_CONTEXT:-262144}
EXPERIMENTAL_CONTEXT=${EXPERIMENTAL_CONTEXT:-76800}
FAST_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}
MEDIUM_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}
LONG_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}
MEDIUM_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
MEDIUM_TENSOR_SPLIT=${MEDIUM_TENSOR_SPLIT:-90,10}
LARGE_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
LARGE_TENSOR_SPLIT=${LARGE_TENSOR_SPLIT:-86,14}
ULTRA_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}${SECONDARY_GPU_DEVICES:+,$SECONDARY_GPU_DEVICES}
ULTRA_TENSOR_SPLIT=${ULTRA_TENSOR_SPLIT:-80,20}
EXPERIMENTAL_GPU_DEVICES=${TEXT_GPU_DEVICES:-0}
@@ -260,7 +262,7 @@ download_models() {
done <<EOF
$FAST_MODEL_FILE|$FAST_MODEL_URL|$FAST_MODEL_SHA256
$MEDIUM_MODEL_FILE|$MEDIUM_MODEL_URL|$MEDIUM_MODEL_SHA256
$LONG_MODEL_FILE|$LONG_MODEL_URL|$LONG_MODEL_SHA256
$LARGE_MODEL_FILE|$LARGE_MODEL_URL|$LARGE_MODEL_SHA256
$ULTRA_MODEL_FILE|$ULTRA_MODEL_URL|$ULTRA_MODEL_SHA256
$EXPERIMENTAL_MODEL_FILE|$EXPERIMENTAL_MODEL_URL|$EXPERIMENTAL_MODEL_SHA256
$VISION_PROJECTOR_FILE|$VISION_PROJECTOR_URL|$VISION_PROJECTOR_SHA256
@@ -329,7 +331,7 @@ build_and_start() {
# secret files and required local artifacts are present.
"$STACK_DIR/platform/mcp/install-tools.sh"
docker compose --env-file "$SECRETS_DIR/stack.env" --profile inference create \
llama-fast llama-medium llama-long llama-ultra llama-experimental
llama-fast llama-medium llama-large llama-ultra llama-experimental
docker compose --env-file "$SECRETS_DIR/stack.env" up -d --build \
profile-controller router open-webui
@@ -343,7 +345,7 @@ build_and_start() {
sleep 2
done
log "Fast-Profil aktivieren und Readiness prüfen"
log "Medium-Profil als Standard aktivieren und Readiness prüfen"
# Use docker exec directly here. Some Compose/Docker combinations return a
# transient HTTP 409 while upgrading the exec stream immediately after a
# freshly built service has been recreated.
@@ -352,7 +354,7 @@ build_and_start() {
import json, os, time, urllib.request
key = os.environ["ROUTER_API_KEY"]
request = urllib.request.Request(
"http://127.0.0.1:8081/fast", method="POST",
"http://127.0.0.1:8081/medium", method="POST",
headers={"Authorization": f"Bearer {key}"})
with urllib.request.urlopen(request, timeout=700) as response:
print(json.dumps(json.load(response), indent=2))
@@ -361,7 +363,7 @@ while time.monotonic() < deadline:
try:
with urllib.request.urlopen("http://127.0.0.1:8081/ready", timeout=5) as response:
if response.status == 200:
print("Router und Fast-Profil sind bereit.")
print("Router und Medium-Standardprofil sind bereit.")
print("INSTALL_READINESS_OK")
break
except Exception:
@@ -370,10 +372,10 @@ while time.monotonic() < deadline:
else:
raise SystemExit("Readiness-Check fehlgeschlagen")
PY
) || die "Fast-Profil konnte nicht aktiviert werden"
) || die "Medium-Standardprofil konnte nicht aktiviert werden"
printf '%s\n' "$activation_output"
grep -Fxq 'INSTALL_READINESS_OK' <<<"$activation_output" || \
die "Fast-Profil lieferte keinen bestätigten Readiness-Marker"
die "Medium-Standardprofil lieferte keinen bestätigten Readiness-Marker"
}
hostnamectl set-hostname "$AI_HOSTNAME"