Cap runaway chat generations in router
This commit is contained in:
@@ -24,6 +24,7 @@ from router_support import ( # noqa: E402
|
||||
load_profile_registry,
|
||||
)
|
||||
from ai_profile_router import ( # noqa: E402
|
||||
_cap_chat_generation,
|
||||
_context_matches,
|
||||
_normalize_chat_image,
|
||||
_normalize_chat_images,
|
||||
@@ -176,6 +177,25 @@ class LlamaCppReasoningTests(unittest.TestCase):
|
||||
self.assertNotIn("chat_template_kwargs", request)
|
||||
|
||||
|
||||
class ChatGenerationLimitTests(unittest.TestCase):
|
||||
def test_missing_limit_gets_platform_cap(self) -> None:
|
||||
request = {"messages": []}
|
||||
self.assertEqual(_cap_chat_generation(request)["max_tokens"], 8192)
|
||||
|
||||
def test_smaller_explicit_limit_is_preserved(self) -> None:
|
||||
request = {"max_tokens": 512}
|
||||
self.assertEqual(_cap_chat_generation(request)["max_tokens"], 512)
|
||||
|
||||
def test_unlimited_and_oversized_values_are_capped(self) -> None:
|
||||
for value in (-1, 0, 999999):
|
||||
with self.subTest(value=value):
|
||||
request = {"max_completion_tokens": value}
|
||||
self.assertEqual(
|
||||
_cap_chat_generation(request)["max_completion_tokens"],
|
||||
8192,
|
||||
)
|
||||
|
||||
|
||||
class RetentionTests(unittest.TestCase):
|
||||
def test_oldest_pairs_are_removed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
|
||||
Reference in New Issue
Block a user