Report llama context limit errors clearly
This commit is contained in:
+19
-1
@@ -19,6 +19,24 @@ class APIError(ValueError):
|
||||
def __init__(self,message,status=400,code='invalid_request_error'):
|
||||
super().__init__(message);self.status=status;self.code=code
|
||||
|
||||
def chat_upstream_error(status, raw):
|
||||
"""Expose known, safe llama.cpp limits without reflecting request content."""
|
||||
try:
|
||||
detail=json.loads(raw)
|
||||
error=detail.get('error',{}) if isinstance(detail,dict) else {}
|
||||
if error.get('type')=='exceed_context_size_error':
|
||||
needed=error.get('n_prompt_tokens');available=error.get('n_ctx')
|
||||
if all(type(value) is int and 0<value<=2097152 for value in (needed,available)):
|
||||
needed_text=f'{needed:,}'.replace(',','.');available_text=f'{available:,}'.replace(',','.')
|
||||
return APIError(
|
||||
f'Die Anfrage benötigt {needed_text} Kontext-Tokens, das aktive Profil bietet {available_text}. '
|
||||
'Kontext im Profil erhöhen oder die Anfrage verkürzen.',
|
||||
400,'context_length_exceeded')
|
||||
except (ValueError,TypeError,AttributeError):
|
||||
pass
|
||||
return APIError('llama.cpp hat die Anfrage abgelehnt. Kontext und Anfrageparameter prüfen.',
|
||||
status if 400<=status<600 else 502,'upstream_error')
|
||||
|
||||
class Endpoint:
|
||||
def __init__(self,root,profiles,worker,scheduler,images,credentials,management_port):
|
||||
self.root=Path(root);self.profiles=profiles;self.worker=worker;self.scheduler=scheduler;self.images=images;self.tts=None;self.stt=None;self.video=None;self.credentials=credentials
|
||||
@@ -287,7 +305,7 @@ class APIHandler(BaseHTTPRequestHandler):
|
||||
conn.request('POST','/v1/chat/completions',body=json.dumps(body).encode(),headers={'Content-Type':'application/json','Authorization':'Bearer '+key})
|
||||
response=conn.getresponse()
|
||||
if response.status!=200:
|
||||
response.read(65536);raise APIError('llama.cpp hat die Anfrage abgelehnt. Kontext und Anfrageparameter prüfen.',response.status if 400<=response.status<600 else 502,'upstream_error')
|
||||
raise chat_upstream_error(response.status,response.read(65536))
|
||||
if not data.get('stream'):
|
||||
raw=response.read(16*1024*1024+1)
|
||||
if len(raw)>16*1024*1024:raise InferenceError('Modellantwort überschreitet 16 MiB.')
|
||||
|
||||
Reference in New Issue
Block a user