diff --git a/compose.yaml b/compose.yaml index c782763..ab124b2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -567,6 +567,10 @@ services: PROFILE_CONTROL_TOKEN: "${CONTROLLER_TOKEN:?CONTROLLER_TOKEN is required}" SWITCH_TIMEOUT: "600" REQUEST_TIMEOUT: "600" + # Last-resort guard for every OpenAI-compatible client. Without a + # request limit llama.cpp uses n_predict=-1 and a reasoning loop can + # consume the complete context before yielding visible output. + MAX_GENERATION_TOKENS: "8192" IMAGE_DIR: /data/images IMAGE_WORKER_URL: http://flux-worker:8086 IMAGE_WORKER_TOKEN: "${CONTROLLER_TOKEN:?CONTROLLER_TOKEN is required}" diff --git a/dev/test_router_support.py b/dev/test_router_support.py index 55dd3b7..4074626 100644 --- a/dev/test_router_support.py +++ b/dev/test_router_support.py @@ -24,6 +24,7 @@ from router_support import ( # noqa: E402 load_profile_registry, ) from ai_profile_router import ( # noqa: E402 + _cap_chat_generation, _context_matches, _normalize_chat_image, _normalize_chat_images, @@ -176,6 +177,25 @@ class LlamaCppReasoningTests(unittest.TestCase): self.assertNotIn("chat_template_kwargs", request) +class ChatGenerationLimitTests(unittest.TestCase): + def test_missing_limit_gets_platform_cap(self) -> None: + request = {"messages": []} + self.assertEqual(_cap_chat_generation(request)["max_tokens"], 8192) + + def test_smaller_explicit_limit_is_preserved(self) -> None: + request = {"max_tokens": 512} + self.assertEqual(_cap_chat_generation(request)["max_tokens"], 512) + + def test_unlimited_and_oversized_values_are_capped(self) -> None: + for value in (-1, 0, 999999): + with self.subTest(value=value): + request = {"max_completion_tokens": value} + self.assertEqual( + _cap_chat_generation(request)["max_completion_tokens"], + 8192, + ) + + class RetentionTests(unittest.TestCase): def test_oldest_pairs_are_removed(self) -> None: with tempfile.TemporaryDirectory() as temp: diff --git a/router/ai_profile_router.py b/router/ai_profile_router.py index 1aa9d02..0f37d80 100755 --- a/router/ai_profile_router.py +++ b/router/ai_profile_router.py @@ -113,6 +113,7 @@ SWITCH_TIMEOUT = float(os.environ.get("SWITCH_TIMEOUT", "600")) # s, Warten REQUEST_TIMEOUT = float(os.environ.get("REQUEST_TIMEOUT", "600")) # s, Read-Timeout Upstream CONNECT_TIMEOUT = float(os.environ.get("CONNECT_TIMEOUT", "10")) # s, Connect-Timeout POLL_INTERVAL = float(os.environ.get("POLL_INTERVAL", "2")) # s, Polling-Intervall +MAX_GENERATION_TOKENS = int(os.environ.get("MAX_GENERATION_TOKENS", "8192")) # --- Bildgenerierung (FLUX.2 [klein] 4B Base) --- LLAMA_SERVICE = os.environ.get("LLAMA_SERVICE", "mike-ai-llama-ui.service") @@ -1166,6 +1167,39 @@ def _normalize_llamacpp_reasoning(data: dict) -> dict: return data +def _cap_chat_generation(data: dict) -> dict: + """Apply a client-independent upper bound to one chat generation. + + Some OpenAI-compatible clients omit both token-limit fields. llama.cpp + interprets that as ``n_predict=-1`` and can remain in hidden reasoning + until the context is exhausted. Smaller explicit limits are preserved. + """ + cap = MAX_GENERATION_TOKENS + if cap <= 0: + return data + + fields = ("max_tokens", "max_completion_tokens") + present = False + for field in fields: + if field not in data: + continue + present = True + value = data[field] + if isinstance(value, bool): + data[field] = cap + continue + try: + parsed = int(value) + except (TypeError, ValueError): + data[field] = cap + continue + data[field] = min(parsed, cap) if parsed > 0 else cap + + if not present: + data["max_tokens"] = cap + return data + + # --------------------------------------------------------------------------- # HTTP-Handler # --------------------------------------------------------------------------- @@ -1975,6 +2009,7 @@ class Handler(BaseHTTPRequestHandler): switch_profile(profile, implicit=True) data = _normalize_llamacpp_reasoning(data) + data = _cap_chat_generation(data) if _request_has_image(data): data = _normalize_chat_images(data)