Report llama context limit errors clearly

test_endpoint.py

@@ -11,7 +11,7 @@ from pathlib import Path
 from types import SimpleNamespace
 from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
 from unittest.mock import Mock
-from endpoint import Endpoint
+from endpoint import Endpoint,chat_upstream_error
 from inference import Scheduler,InferenceError
 
 class FakeWorker:
@@ -32,6 +32,19 @@ class Upstream(BaseHTTPRequestHandler):
   else:
    body=json.dumps(dict(id='test',model=data['model'],choices=[{'message':{'role':'assistant','content':'synthetic'}}])).encode();self.send_response(200);self.send_header('Content-Length',str(len(body)));self.end_headers();self.wfile.write(body)
 
+class UpstreamErrorTests(unittest.TestCase):
+ def test_context_limit_is_actionable_without_echoing_upstream_text(self):
+  raw=json.dumps({'error':{'type':'exceed_context_size_error','message':'private request text',
+                           'n_prompt_tokens':13523,'n_ctx':8192}}).encode()
+  error=chat_upstream_error(400,raw)
+  self.assertEqual(error.code,'context_length_exceeded')
+  self.assertIn('13.523',str(error));self.assertIn('8.192',str(error))
+  self.assertNotIn('private request text',str(error))
+ def test_unknown_upstream_errors_stay_generic(self):
+  error=chat_upstream_error(400,b'{"error":{"message":"private request text"}}')
+  self.assertEqual(error.code,'upstream_error')
+  self.assertNotIn('private request text',str(error))
+
 class EndpointTests(unittest.TestCase):
  def setUp(self):
   self.tmp=tempfile.TemporaryDirectory();self.worker=FakeWorker();self.worker.http=ThreadingHTTPServer(('127.0.0.1',0),Upstream);self.worker.http.worker=self.worker;threading.Thread(target=self.worker.http.serve_forever,daemon=True).start()