michael/AI-Profile-Router
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Simplify Athena runtime and document current architecture

Browse Source
This commit is contained in:
Mikei386
2026-08-30 22:11:26 +02:00
parent 9bc7d9803a
commit 0b927d47b9
56 changed files with 1276 additions and 4712 deletions
Download Patch File Download Diff File Expand all files Collapse all files
router/ai_profile_router.py
+154 -18
View File
@@ -18,7 +18,7 @@ Virtuelle Modelle: qwen-fast, qwen-medium, qwen-large, qwen-ultra,
Kommandos: POST /fast, /medium, /large, /ultra, /uncensored
GET /status (Zustand)
Bildgenerierung (FLUX.2 [klein] 4B Base):
Bildgenerierung (Z-Image-Turbo):
POST /v1/images/generations (OpenAI-kompatibel)
GET /images (Liste)
GET /images/<datei> (PNG-Download)
@@ -37,7 +37,7 @@ Der Router leitet /v1/audio/speech und /v1/audio/transcriptions
per HTTP an die Worker weiter.
Der Router agiert als Modell-Orchestrator: vor der Generierung wird
llama.cpp gestoppt, der Bild-Worker lädt FLUX, generiert und entlädt
llama.cpp gestoppt, der Bild-Worker lädt Z-Image, generiert und entlädt
das Modell wieder; danach wird das vorherige Qwen-Profil wiederher-
gestellt und erst dann geantwortet (try/finally – Qwen wird auch bei
Fehlgeschlagener Generierung wiederhergestellt).
@@ -116,7 +116,7 @@ CONNECT_TIMEOUT = float(os.environ.get("CONNECT_TIMEOUT", "10")) # s, Connect
POLL_INTERVAL = float(os.environ.get("POLL_INTERVAL", "2")) # s, Polling-Intervall
MAX_GENERATION_TOKENS = int(os.environ.get("MAX_GENERATION_TOKENS", "8192"))
# --- Bildgenerierung (FLUX.2 [klein] 4B Base) ---
# --- Bildgenerierung (Z-Image-Turbo) ---
LLAMA_SERVICE = os.environ.get("LLAMA_SERVICE", "mike-ai-llama-ui.service")
SYSTEMCTL_BIN = os.environ.get("SYSTEMCTL_BIN", "systemctl")
IMAGE_WORKER = os.environ.get(
@@ -125,6 +125,7 @@ IMAGE_PYTHON = os.environ.get(
"IMAGE_PYTHON", "/opt/mike-ai/ai-profile-router/venv/bin/python")
IMAGE_WORKER_URL = os.environ.get("IMAGE_WORKER_URL", "").rstrip("/")
IMAGE_WORKER_TOKEN = os.environ.get("IMAGE_WORKER_TOKEN", "").strip()
IMAGE_MODEL_NAME = os.environ.get("IMAGE_MODEL_NAME", "Z-Image-Turbo")
IMAGE_DIR = os.environ.get(
"IMAGE_DIR", "/opt/mike-ai/ai-profile-router/images")
IMAGE_WORKER_LOG = os.environ.get(
@@ -144,8 +145,7 @@ CHAT_IMAGE_MAX_BYTES = int(os.environ.get(
CHAT_IMAGE_ALLOW_REMOTE_URLS = os.environ.get(
"CHAT_IMAGE_ALLOW_REMOTE_URLS", "false").lower() in {"1", "true", "yes"}
# Erlaubte Auflösungen (Breite x Höhe). FLUX.2 klein ist für 1 MP
# ausgelegt; 1920x1088 (≈2 MP) wird zusätzlich unterstützt.
# Erlaubte Auflösungen (Breite x Höhe).
IMAGE_SIZES = {
"1024x1024": (1024, 1024),
"1536x1024": (1536, 1024),
@@ -153,10 +153,8 @@ IMAGE_SIZES = {
"1920x1088": (1920, 1088),
"1088x1920": (1088, 1920),
}
# FLUX.2 Klein Distilled ist fest auf vier Schritte und Guidance 1.0
# destilliert. Qualitätsstufen bleiben aus OpenAI-Kompatibilitätsgründen
# akzeptiert, ändern aber bewusst nicht die offiziellen Sampling-Werte.
IMAGE_QUALITY = {"standard": 4, "high": 4}
# Z-Image-Turbo nutzt neun Scheduler-Schritte (acht DiT-Forwards) ohne CFG.
IMAGE_QUALITY = {"standard": 9, "high": 9}
IMAGE_DEFAULT_QUALITY = "standard"
IMAGE_MAX_N = 4
@@ -433,6 +431,136 @@ def upstream_status() -> dict:
"ctx": (m.get("meta") or {}).get("n_ctx")}
_TELEMETRY_LOCK = threading.Lock()
_TELEMETRY_AT = 0.0
_TELEMETRY_CACHE: dict = {}
def _upstream_read(path: str, *, timeout: float = 1.5) -> tuple[int, bytes]:
"""Read a bounded, read-only llama.cpp telemetry endpoint."""
conn = http.client.HTTPConnection(UPSTREAM_HOST, UPSTREAM_PORT,
timeout=timeout)
try:
conn.request("GET", path, headers={"Accept": "application/json,text/plain"})
resp = conn.getresponse()
return resp.status, resp.read(2 * 1024 * 1024)
finally:
conn.close()
def _parse_prometheus_metrics(raw: str) -> dict:
wanted = {
"llamacpp:prompt_tokens_total",
"llamacpp:prompt_tokens_cached_total",
"llamacpp:prompt_seconds_total",
"llamacpp:tokens_predicted_total",
"llamacpp:tokens_predicted_seconds_total",
"llamacpp:n_decode_total",
"llamacpp:n_tokens_max",
"llamacpp:spec_decode_num_draft_tokens_total",
"llamacpp:spec_decode_num_accepted_tokens_total",
"llamacpp:spec_decode_num_drafts_total",
"llamacpp:prompt_tokens_seconds",
"llamacpp:predicted_tokens_seconds",
"llamacpp:requests_processing",
"llamacpp:requests_deferred",
"llamacpp:n_busy_slots_per_decode",
}
result: dict[str, int | float] = {}
for line in raw.splitlines():
if not line or line.startswith("#") or " " not in line:
continue
name, value = line.rsplit(None, 1)
if "{" in name or name not in wanted:
continue
try:
number = float(value)
result[name.removeprefix("llamacpp:")] = (
int(number) if number.is_integer() else number
)
except ValueError:
continue
return result
def upstream_telemetry() -> dict:
"""Compact llama.cpp slots, rates, cache and MTP telemetry.
The result is cached briefly because the dashboard refreshes every second.
Failures never affect inference or the normal router status response.
"""
global _TELEMETRY_AT, _TELEMETRY_CACHE
now = time.monotonic()
with _TELEMETRY_LOCK:
if now - _TELEMETRY_AT < 0.75 and _TELEMETRY_CACHE:
return _TELEMETRY_CACHE
result: dict = {"available": False, "slots": [], "metrics": {}}
errors: dict[str, str] = {}
try:
status, body = _upstream_read("/slots")
if status == 200:
raw_slots = json.loads(body)
for slot in raw_slots if isinstance(raw_slots, list) else []:
next_token = (slot.get("next_token") or [{}])[0]
params = slot.get("params") or {}
prompt = int(slot.get("n_prompt_tokens") or 0)
decoded = int(next_token.get("n_decoded") or 0)
n_ctx = int(slot.get("n_ctx") or 0)
result["slots"].append({
"id": slot.get("id"),
"task_id": slot.get("id_task"),
"processing": bool(slot.get("is_processing")),
"speculative": bool(slot.get("speculative")),
"n_ctx": n_ctx,
"prompt_tokens": prompt,
"prompt_processed": int(slot.get("n_prompt_tokens_processed") or 0),
"prompt_cached": int(slot.get("n_prompt_tokens_cache") or 0),
"decoded_tokens": decoded,
"context_used": min(n_ctx, prompt + decoded) if n_ctx else prompt + decoded,
"remaining_generation": next_token.get("n_remain"),
"max_tokens": params.get("max_tokens", params.get("n_predict")),
"temperature": params.get("temperature"),
"stream": params.get("stream"),
})
else:
errors["slots"] = f"HTTP {status}"
except (OSError, ValueError, KeyError, TypeError, http.client.HTTPException) as exc:
errors["slots"] = str(exc)
try:
status, body = _upstream_read("/metrics")
if status == 200:
result["metrics"] = _parse_prometheus_metrics(
body.decode("utf-8", "replace")
)
else:
errors["metrics"] = f"HTTP {status}"
except (OSError, ValueError, http.client.HTTPException) as exc:
errors["metrics"] = str(exc)
try:
status, body = _upstream_read("/props")
if status == 200:
props = json.loads(body)
result["props"] = {
"total_slots": props.get("total_slots"),
"model_alias": props.get("model_alias"),
"model_ftype": props.get("model_ftype"),
"model_path": props.get("model_path"),
"modalities": props.get("modalities") or {},
"default_context": ((props.get("default_generation_settings") or {})
.get("n_ctx")),
}
else:
errors["props"] = f"HTTP {status}"
except (OSError, ValueError, TypeError, http.client.HTTPException) as exc:
errors["props"] = str(exc)
result["available"] = bool(result["slots"] or result["metrics"])
if errors:
result["errors"] = errors
_TELEMETRY_CACHE = result
_TELEMETRY_AT = now
return result
# ---------------------------------------------------------------------------
# Profile
# ---------------------------------------------------------------------------
@@ -614,7 +742,7 @@ def switch_profile(profile: str, implicit: bool = False) -> None:
# ---------------------------------------------------------------------------
# Bildgenerierung (FLUX.2 [klein] 4B Base)
# Bildgenerierung (Z-Image-Turbo)
# ---------------------------------------------------------------------------
class _Worker:
@@ -753,8 +881,9 @@ class _RemoteWorker:
deadline = time.monotonic() + IMAGE_START_TIMEOUT
while time.monotonic() < deadline:
try:
self._request("GET", "/health", timeout=3)
health = self._request("GET", "/health", timeout=3)
self.running = True
self.model_loaded = bool(health.get("model_loaded"))
return
except (OSError, urllib.error.URLError, TimeoutError, RuntimeError):
time.sleep(1)
@@ -804,7 +933,7 @@ def _vram_used_mib() -> int | None:
def _wait_vram_free(threshold_mib: int = 1000,
timeout: float | None = None) -> None:
"""Wartet, bis der VRAM unter threshold_mib fällt (FLUX entladen).
"""Wartet, bis der VRAM unter threshold_mib fällt (Bildmodell entladen).
Wird nach dem Beenden des Bild-Workers aufgerufen, um sicherzustellen,
dass der VRAM (inkl. CUDA-Kontext) frei ist, bevor Qwen neu startet.
@@ -937,7 +1066,7 @@ def generate_image(prompt: str, width: int, height: int, steps: int,
"guidance": guidance,
"quality": quality,
"seconds": resp.get("seconds"),
"model": "FLUX.2-klein-4B",
"model": IMAGE_MODEL_NAME,
"created": time.strftime("%Y-%m-%dT%H:%M:%S"),
}
meta_path = os.path.join(IMAGE_DIR, filename[:-4] + ".json")
@@ -1475,10 +1604,17 @@ class Handler(BaseHTTPRequestHandler):
and bool(up.get("model"))),
"active_chats": active_chats,
},
"llama_telemetry": (upstream_telemetry() if up["reachable"] else {
"available": False,
"slots": [],
"metrics": {},
"errors": {"upstream": up.get("error", "not reachable")},
}),
"image": {
"phase": img.phase,
"worker": "running" if (img.worker and img.worker.alive())
else "stopped",
"model": IMAGE_MODEL_NAME if img.phase != "idle" else None,
"model_loaded": bool(img.worker and img.worker.model_loaded),
"last_image": img.last_image,
"last_seconds": img.last_seconds,
@@ -1543,19 +1679,19 @@ class Handler(BaseHTTPRequestHandler):
"invalid_request_error", "invalid_quality")
return
steps = data.get("steps", IMAGE_QUALITY[quality])
if not isinstance(steps, int) or isinstance(steps, bool) or steps != 4:
self._send_error(400, "FLUX.2 Klein Distilled erfordert 'steps'=4",
if not isinstance(steps, int) or isinstance(steps, bool) or steps != 9:
self._send_error(400, "Z-Image-Turbo erfordert 'steps'=9",
"invalid_request_error", "invalid_steps")
return
guidance = data.get("guidance", 1.0)
guidance = data.get("guidance", 0.0)
try:
guidance = float(guidance)
except (TypeError, ValueError):
self._send_error(400, "'guidance' muss eine Zahl sein",
"invalid_request_error", "invalid_guidance")
return
if guidance != 1.0:
self._send_error(400, "FLUX.2 Klein Distilled erfordert 'guidance'=1.0",
if guidance != 0.0:
self._send_error(400, "Z-Image-Turbo erfordert 'guidance'=0.0",
"invalid_request_error", "invalid_guidance")
return