Add owned OpenAI endpoint and coordinated native model switching
This commit is contained in:
Mikei386
+221
@@ -0,0 +1,221 @@
|
||||
"""Separate authenticated OpenAI-compatible API; only explicitly enabled Deck profiles."""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
|
||||
from inference import InferenceError
|
||||
|
||||
class APIError(ValueError):
|
||||
def __init__(self,message,status=400,code='invalid_request_error'):
|
||||
super().__init__(message);self.status=status;self.code=code
|
||||
|
||||
class Endpoint:
|
||||
def __init__(self,root,profiles,worker,scheduler,images,credentials,management_port):
|
||||
self.root=Path(root);self.profiles=profiles;self.worker=worker;self.scheduler=scheduler;self.images=images;self.credentials=credentials
|
||||
self.management_port=management_port;self.lock=threading.RLock();self.http=None;self.thread=None;self.state='stopped';self.error=None;self.inflight=0
|
||||
self.allowed_ports=[int(p) for p in os.environ.get('DECK_API_PORTS','').split(',') if p]
|
||||
self.config=dict(port=self.allowed_ports[0] if self.allowed_ports else 8120,enabled_profiles=[],autostart=False)
|
||||
path=self.root/'endpoint.json'
|
||||
if path.exists():self.config.update(json.loads(path.read_text()))
|
||||
def persist(self):
|
||||
self.root.mkdir(parents=True,exist_ok=True,mode=0o700)
|
||||
path=self.root/'endpoint.tmp';path.write_text(json.dumps(self.config));path.replace(self.root/'endpoint.json')
|
||||
def rows(self):
|
||||
rows=self.profiles.status()['profiles']
|
||||
with self.lock:enabled=set(self.config['enabled_profiles'])
|
||||
return [dict(p,enabled=p['id'] in enabled) for p in rows]
|
||||
def status(self):
|
||||
rows=self.rows();worker=self.worker.status();job=self.images.status()['job'];counts={}
|
||||
for key,kind in [('llm','chat'),('image','image'),('tts','tts'),('stt','stt')]:
|
||||
subset=[p for p in rows if p['kind']==kind]
|
||||
counts[key]=dict(enabled=sum(p['enabled'] for p in subset),available=sum(p['enabled'] and p['runnable'] for p in subset),loaded=bool(worker['state']=='ready' and kind=='chat') if kind=='chat' else bool(kind=='image' and job and job['state']=='running'),supported=kind in ('chat','image'))
|
||||
scheduler_status=self.scheduler.status()
|
||||
with self.lock:return dict(state=self.state,reachable=bool(self.thread and self.thread.is_alive() and self.state=='running'),port=self.config['port'],bind=os.environ.get('DECK_API_BIND','127.0.0.1'),base_url=f"http://127.0.0.1:{self.config['port']}/v1",allowed_ports=self.allowed_ports,error=self.error,counts=counts,worker=worker,scheduler=scheduler_status,profiles=[dict(id=p['id'],name=p['name'],kind=p['kind'],enabled=p['enabled'],runnable=p['runnable'],blockers=p['blockers']) for p in rows],active_requests=self.inflight)
|
||||
def configure(self,data):
|
||||
if set(data)!={'port'} or type(data['port']) is not int or not 1024<=data['port']<=65535:raise ValueError('Port zwischen 1024 und 65535 erforderlich.')
|
||||
port=data['port']
|
||||
with self.lock:
|
||||
if self.state!='stopped':raise ValueError('Endpunkt zuerst vollständig stoppen.')
|
||||
if port==self.management_port:raise ValueError('Der Verwaltungsport ist bereits belegt.')
|
||||
if self.allowed_ports and port not in self.allowed_ports:raise ValueError('Dieser Port ist im Docker-Installer nicht freigegeben.')
|
||||
try:
|
||||
with socket.socket() as sock:sock.bind((os.environ.get('DECK_API_BIND','127.0.0.1'),port))
|
||||
except OSError:raise ValueError('Port ist bereits belegt.') from None
|
||||
self.config['port']=port;self.persist()
|
||||
return self.status()
|
||||
def enable(self,data):
|
||||
if set(data)!={'id','enabled'} or type(data['enabled']) is not bool:raise ValueError('Profil-ID und Aktivierung erforderlich.')
|
||||
row=next((p for p in self.rows() if p['id']==data['id']),None)
|
||||
if not row:raise ValueError('Profil nicht gefunden.')
|
||||
if data['enabled'] and not row['runnable']:raise ValueError('Profil nicht ausführbar: '+' '.join(row['blockers']))
|
||||
with self.lock:
|
||||
enabled=set(self.config['enabled_profiles'])
|
||||
if data['enabled']:enabled.add(data['id'])
|
||||
else:enabled.discard(data['id'])
|
||||
self.config['enabled_profiles']=sorted(enabled);self.persist()
|
||||
return self.status()
|
||||
def start(self):
|
||||
with self.lock:
|
||||
if self.state=='running':return {'state':'running','port':self.config['port']}
|
||||
if self.state!='stopped':raise ValueError('Endpunkt wird noch gestoppt.')
|
||||
record=self.credentials.read()
|
||||
if not record or not record.get('api_token_hash'):raise ValueError('Zuerst unter Zugang & API einen API-Token einrichten.')
|
||||
if self.allowed_ports and self.config['port'] not in self.allowed_ports:raise ValueError('Gespeicherter Port ist nicht im Docker-Installer freigegeben.')
|
||||
if self.config['port']==self.management_port:raise ValueError('Verwaltungsport kann nicht als API-Port verwendet werden.')
|
||||
try:http=APIHTTPServer((os.environ.get('DECK_API_BIND','127.0.0.1'),self.config['port']),APIHandler)
|
||||
except OSError:raise ValueError('API-Port bereits belegt; kein anderer Dienst wurde verändert.') from None
|
||||
http.endpoint=self;self.http=http;self.state='running';self.error=None
|
||||
self.thread=threading.Thread(target=http.serve_forever,daemon=True);self.thread.start()
|
||||
self.config['autostart']=True;self.persist()
|
||||
return self.status()
|
||||
def stop(self):
|
||||
with self.lock:
|
||||
self.config['autostart']=False;self.persist()
|
||||
if self.state=='running':
|
||||
self.state='stopping';threading.Thread(target=self._drain,daemon=True).start()
|
||||
return self.status()
|
||||
def _drain(self):
|
||||
http=self.http
|
||||
if http:http.shutdown();http.server_close()
|
||||
while True:
|
||||
with self.lock:pending=self.inflight
|
||||
if not pending and self.scheduler.unload_idle():break
|
||||
time.sleep(.1)
|
||||
with self.lock:self.http=None;self.thread=None;self.state='stopped'
|
||||
def close(self):
|
||||
# Process shutdown does not change the user's autostart preference.
|
||||
with self.lock:self.state='stopping';http=self.http
|
||||
if http:http.shutdown();http.server_close()
|
||||
self.images.stop();self.worker.stop()
|
||||
def allowed(self):
|
||||
with self.lock:return self.state=='running'
|
||||
def authenticate(self,header):
|
||||
if not header.startswith('Bearer ') or len(header)>300:return False
|
||||
record=self.credentials.read()
|
||||
return bool(record and record.get('api_token_hash') and secrets.compare_digest(hashlib.sha256(header[7:].encode()).hexdigest(),record['api_token_hash']))
|
||||
def find_profile(self,name,kind):
|
||||
if not isinstance(name,str):raise APIError('model muss den API-Namen eines aktivierten Profils enthalten.')
|
||||
row=next((p for p in self.rows() if p['name']==name and p['kind']==kind and p['enabled']),None)
|
||||
if not row:raise APIError('Modellprofil nicht aktiviert oder unbekannt.',404,'model_not_found')
|
||||
if not row['runnable']:raise APIError('Profil derzeit nicht ausführbar: '+' '.join(row['blockers']),503,'model_unavailable')
|
||||
return row
|
||||
def model_list(self):
|
||||
return dict(object='list',data=[dict(id=p['name'],object='model',created=int(p['updated_at']),owned_by='athena-deck') for p in self.rows() if p['enabled'] and p['runnable']])
|
||||
|
||||
class APIHTTPServer(ThreadingHTTPServer):
|
||||
daemon_threads=True
|
||||
def __init__(self,*args,**kwargs):
|
||||
self.admission=threading.BoundedSemaphore(32);super().__init__(*args,**kwargs)
|
||||
def process_request(self,request,address):
|
||||
if not self.admission.acquire(blocking=False):self.shutdown_request(request);return
|
||||
try:super().process_request(request,address)
|
||||
except Exception:self.admission.release();raise
|
||||
def process_request_thread(self,*args):
|
||||
try:super().process_request_thread(*args)
|
||||
finally:self.admission.release()
|
||||
def handle_error(self,*args):pass # No request bodies, prompts, or traces in logs.
|
||||
|
||||
class APIHandler(BaseHTTPRequestHandler):
|
||||
protocol_version='HTTP/1.1'
|
||||
def setup(self):super().setup();self.connection.settimeout(15);self.sent=False
|
||||
def log_message(self,*args):pass
|
||||
def send(self,payload,status=200):
|
||||
body=json.dumps(payload).encode();self.sent=True
|
||||
self.send_response(status);self.send_header('Content-Type','application/json');self.send_header('Content-Length',str(len(body)));self.send_header('Cache-Control','no-store');self.send_header('Connection','close');self.end_headers();self.wfile.write(body);self.close_connection=True
|
||||
def failure(self,exc):
|
||||
if self.sent:return
|
||||
self.send({'error':{'message':str(exc),'type':getattr(exc,'code','server_error'),'param':None,'code':getattr(exc,'code','worker_unavailable')}},getattr(exc,'status',503))
|
||||
def do_GET(self):self.route()
|
||||
def do_POST(self):self.route()
|
||||
def route(self):
|
||||
ep=self.server.endpoint;admitted=False
|
||||
try:
|
||||
if not ep.authenticate(self.headers.get('Authorization','')):raise APIError('Gültiger API-Bearer-Token erforderlich.',401,'invalid_api_key')
|
||||
if self.headers.get('Origin'):raise APIError('Browserzugriff erfolgt über die Deck-Verwaltung; keine Cross-Origin-API-Freigabe.',403)
|
||||
with ep.lock:
|
||||
if not ep.allowed():raise APIError('Endpunkt wird gestoppt.',503,'endpoint_stopping')
|
||||
ep.inflight+=1;admitted=True
|
||||
if self.command=='GET' and self.path=='/v1/models':return self.send(ep.model_list())
|
||||
if self.command=='GET' and self.path=='/health':return self.send({'status':'ok','service':'athena-deck-api'})
|
||||
if self.command!='POST':raise APIError('Route nicht gefunden.',404)
|
||||
if self.path in ('/v1/audio/speech','/v1/audio/transcriptions'):raise APIError('Für TTS/STT ist noch keine Deck-Laufzeit eingerichtet.',501,'not_implemented')
|
||||
if self.path not in ('/v1/chat/completions','/v1/images/generations'):raise APIError('Route nicht implementiert.',404)
|
||||
if self.headers.get('Transfer-Encoding'):raise APIError('Chunked Upload wird nicht unterstützt.')
|
||||
try:length=int(self.headers.get('Content-Length','0'))
|
||||
except ValueError:raise APIError('Ungültige Content-Length.') from None
|
||||
if not 0<length<=1024*1024 or self.headers.get('Content-Type','').split(';')[0]!='application/json':raise APIError('JSON-Anfrage bis 1 MiB erforderlich.',413)
|
||||
try:data=json.loads(self.rfile.read(length),parse_constant=lambda _:(_ for _ in ()).throw(ValueError()))
|
||||
except (ValueError,UnicodeError):raise APIError('Ungültiges JSON.') from None
|
||||
if not isinstance(data,dict):raise APIError('JSON-Objekt erforderlich.')
|
||||
if self.path=='/v1/chat/completions':return self.chat(ep,data)
|
||||
return self.image(ep,data)
|
||||
except (APIError,InferenceError) as exc:
|
||||
if isinstance(exc,InferenceError):
|
||||
with ep.lock:ep.error=str(exc)
|
||||
self.failure(exc)
|
||||
except (BrokenPipeError,ConnectionResetError):pass
|
||||
except Exception:self.failure(APIError('Worker nicht erreichbar oder Zeitlimit überschritten.',503,'worker_unavailable'))
|
||||
finally:
|
||||
self.close_connection=True
|
||||
if admitted:
|
||||
with ep.lock:ep.inflight-=1
|
||||
def chat(self,ep,data):
|
||||
supported={'model','messages','stream','stream_options','temperature','top_p','top_k','max_tokens','max_completion_tokens','stop','seed','tools','tool_choice','parallel_tool_calls','response_format','presence_penalty','frequency_penalty','logprobs','top_logprobs','user','n'}
|
||||
if set(data)-supported:raise APIError('Nicht unterstützte Chat-Felder: '+', '.join(sorted(set(data)-supported)))
|
||||
if type(data.get('stream',False)) is not bool:raise APIError('stream muss true oder false sein.')
|
||||
if data.get('n',1)!=1:raise APIError('Zunächst wird n=1 unterstützt.')
|
||||
messages=data.get('messages')
|
||||
if not isinstance(messages,list) or not messages or any(not isinstance(m,dict) for m in messages):raise APIError('messages muss eine nichtleere Liste sein.')
|
||||
for message in messages:
|
||||
content=message.get('content')
|
||||
if isinstance(content,list) and any(not isinstance(c,dict) or c.get('type')!='text' for c in content):raise APIError('Dieses Chatprofil unterstützt derzeit nur Text, keinen Vision-Projektor.')
|
||||
profile=ep.find_profile(data.get('model'),'chat')
|
||||
# Re-resolve after waiting so edits/disable cannot silently launch a stale profile.
|
||||
key=('chat',profile['id'],profile['revision'])
|
||||
def allowed():return ep.allowed() and any(p['id']==profile['id'] and p['revision']==profile['revision'] and p['enabled'] for p in ep.rows())
|
||||
with ep.scheduler.lease(key,profile['parameters']['slots'],lambda:ep.worker.ensure(profile),allowed=allowed):
|
||||
body=dict(data)
|
||||
for field in ('temperature','top_p','top_k'):body.setdefault(field,profile['parameters'][field])
|
||||
conn,key=ep.worker.connect()
|
||||
try:
|
||||
conn.request('POST','/v1/chat/completions',body=json.dumps(body).encode(),headers={'Content-Type':'application/json','Authorization':'Bearer '+key})
|
||||
response=conn.getresponse()
|
||||
if response.status!=200:
|
||||
response.read(65536);raise APIError('llama.cpp hat die Anfrage abgelehnt. Kontext und Anfrageparameter prüfen.',response.status if 400<=response.status<600 else 502,'upstream_error')
|
||||
if not data.get('stream'):
|
||||
raw=response.read(16*1024*1024+1)
|
||||
if len(raw)>16*1024*1024:raise InferenceError('Modellantwort überschreitet 16 MiB.')
|
||||
return self.send(json.loads(raw))
|
||||
self.sent=True;self.connection.settimeout(30)
|
||||
self.send_response(200);self.send_header('Content-Type','text/event-stream');self.send_header('Cache-Control','no-store');self.send_header('Connection','close');self.end_headers()
|
||||
deadline=time.monotonic()+600
|
||||
while time.monotonic()<deadline:
|
||||
chunk=response.read1(8192)
|
||||
if not chunk:return
|
||||
self.wfile.write(chunk);self.wfile.flush()
|
||||
raise InferenceError('Zeitlimit der Streaming-Antwort überschritten.')
|
||||
finally:conn.close()
|
||||
def image(self,ep,data):
|
||||
if set(data)-{'model','prompt','n','size','response_format','user'}:raise APIError('Nicht unterstützte Bildparameter. Schritte/Guidance/Seed stehen im Profil.')
|
||||
if data.get('n',1)!=1 or data.get('response_format','b64_json')!='b64_json':raise APIError('Unterstützt werden n=1 und response_format=b64_json.')
|
||||
profile=ep.find_profile(data.get('model'),'image');params=profile['parameters']
|
||||
if data.get('size',f"{params['width']}x{params['height']}")!=f"{params['width']}x{params['height']}":raise APIError('size muss der im Profil gespeicherten Auflösung entsprechen.')
|
||||
def allowed():return ep.allowed() and any(p['id']==profile['id'] and p['revision']==profile['revision'] and p['enabled'] for p in ep.rows())
|
||||
with ep.scheduler.lease(('image',),allowed=allowed):
|
||||
return self.generate_image(ep,profile,data)
|
||||
def generate_image(self,ep,profile,data):
|
||||
job=ep.images.start(profile['id'],data.get('prompt'),reserved=True)
|
||||
deadline=time.monotonic()+2100
|
||||
while time.monotonic()<deadline:
|
||||
current=ep.images.status()['job']
|
||||
if not current or current['id']!=job['id']:raise InferenceError('Bildauftrag nicht mehr verfügbar.')
|
||||
if current['state']=='complete':return self.send({'created':int(time.time()),'data':[{'b64_json':base64.b64encode(ep.images.image(job['id'])).decode()}]})
|
||||
if current['state']!='running':raise InferenceError(current['phase'])
|
||||
time.sleep(.25)
|
||||
ep.images.stop();raise InferenceError('Zeitlimit der Bildgenerierung überschritten.')
|
||||
Reference in New Issue
Block a user