265 lines
18 KiB
Python
265 lines
18 KiB
Python
"""Public Hub catalogue and bounded, serial downloads; independent of Docker/systemd."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path, PurePosixPath
|
|
import re
|
|
import shutil
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import urllib.parse
|
|
import urllib.request
|
|
import urllib.error
|
|
|
|
KINDS={'chat':'text-generation','image':'text-to-image','audio':'text-to-speech','stt':'automatic-speech-recognition','music':'text-to-audio','voice':'audio-to-audio','video':'text-to-video'}
|
|
|
|
def safe_url(url):
|
|
p=urllib.parse.urlsplit(url)
|
|
if p.scheme!='https' or p.username or p.password or p.port not in (None,443) or not any(p.hostname==d or (p.hostname or '').endswith('.'+d) for d in ('huggingface.co','hf.co','xethub.hf.co')):
|
|
raise ValueError('Downloadziel außerhalb der erlaubten Hugging-Face-Domains.')
|
|
return url
|
|
|
|
class Redirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
redirected=super().redirect_request(req,fp,code,msg,headers,safe_url(newurl))
|
|
if redirected and urllib.parse.urlsplit(req.full_url).netloc != urllib.parse.urlsplit(newurl).netloc:
|
|
redirected.remove_header('Authorization')
|
|
return redirected
|
|
|
|
def remote(url,token=None):
|
|
headers={'User-Agent':'Athena-Deck/0.5'}
|
|
if token and urllib.parse.urlsplit(safe_url(url)).netloc=='huggingface.co':headers['Authorization']='Bearer '+token
|
|
try:
|
|
return urllib.request.build_opener(Redirect()).open(urllib.request.Request(safe_url(url),headers=headers),timeout=20)
|
|
except urllib.error.HTTPError as exc:
|
|
code=exc.code;exc.close()
|
|
messages={401:'Hugging Face: Token fehlt oder ist ungültig. Unter Einstellungen → Zugang prüfen.',403:'Hugging Face verweigert den Zugriff. Modellfreigabe auf der Modellkarte beantragen und Leserechte des Tokens prüfen.',404:'Hugging-Face-Repository oder Datei nicht gefunden; bei privaten Repositories Token und Leserechte prüfen.'}
|
|
raise ValueError(messages.get(code,'Hugging-Face-Anfrage fehlgeschlagen (HTTP '+str(code)+').')) from None
|
|
|
|
def metadata(path,token=None):
|
|
with remote('https://huggingface.co'+path,token) as r:
|
|
raw=r.read(8*1024*1024+1)
|
|
if len(raw)>8*1024*1024:raise ValueError('Metadaten zu groß; Repository wird noch nicht unterstützt.')
|
|
return json.loads(raw)
|
|
|
|
def repo_id(value):
|
|
if not isinstance(value,str) or not re.fullmatch(r'[A-Za-z0-9_-][A-Za-z0-9_.-]{0,95}/[A-Za-z0-9_-][A-Za-z0-9_.-]{0,95}',value):raise ValueError('Ungültige Repository-ID.')
|
|
return value
|
|
|
|
def file_role(filename):
|
|
"""One classification shared by library UI and profile validation."""
|
|
name=filename.lower();parts=PurePosixPath(name).parts
|
|
if re.search(r'(?:^|[/_-])mmproj(?:[/_.-]|$)',name):
|
|
role,label=('audio_projector','Audio-Projektor') if 'qwen3-asr' in name else ('vision_projector','Vision-Projektor')
|
|
elif 'speech_tokenizer' in parts:
|
|
role,label='auxiliary','Sprach-Tokenizer'
|
|
elif any(p in ('text_encoder','text_encoders') for p in parts) or re.search(r'(?:^|[/_-])(?:text.encoder|qwen3vl|t5xxl|clip_l|clip_g)(?:[/_.-]|$)',name):
|
|
role,label='text_encoder','Textencoder'
|
|
elif 'vae' in parts or re.search(r'(?:^|[/_-])vae(?:[/_.-]|$)',name):
|
|
role,label='vae','VAE'
|
|
elif any(p in ('latent_upscale_models','model_patches','loras') for p in parts):
|
|
role,label='auxiliary','Zusatzkomponente'
|
|
elif re.search(r'(?:^|[/_-])(?:lora|adapter)(?:[/_.-]|$)',name):
|
|
role,label='auxiliary','Zusatzkomponente'
|
|
elif name.endswith(('.gguf','.safetensors')):
|
|
role,label='model','Modelldatei'
|
|
else:
|
|
role,label='configuration','Konfiguration'
|
|
return dict(role=role,role_label=label,profile_eligible=role=='model')
|
|
|
|
def is_derived_model(model):
|
|
"""Use publisher metadata, never infer lineage from a repository name."""
|
|
card=model.get('cardData') or {}
|
|
if isinstance(card,dict) and (card.get('base_model') or card.get('base_model_relation')):return True
|
|
tags=model.get('tags') or []
|
|
return any(isinstance(tag,str) and (tag.startswith('base_model:') or tag.lower() in {'lora','peft','adapter','merge','mergekit','finetune','fine-tuned','quantized','gguf','gptq','awq'}) for tag in tags)
|
|
|
|
def capabilities(model):
|
|
tags={t.lower() for t in (model.get('tags') or []) if isinstance(t,str)}
|
|
vision=bool(tags & {'image-text-to-text','image-to-text','vision','multimodal'}) or model.get('pipeline_tag') in ('image-text-to-text','image-to-text')
|
|
files=any(file_role(x.get('rfilename',''))['role']=='vision_projector' for x in (model.get('siblings') or []))
|
|
tools=bool(tags & {'tool-use','tool-calling','function-calling','function_calling'})
|
|
return dict(vision='Hinweis laut Hub' if vision else 'Projektordatei vorhanden' if files else 'Unbekannt',tools='Hinweis laut Hub' if tools else 'Unbekannt')
|
|
|
|
def runtime_metadata(model):
|
|
"""Persist only bounded, non-secret Hub hints; never treat them as a runnable recipe."""
|
|
config=model.get('config') if isinstance(model.get('config'),dict) else {}
|
|
architectures=config.get('architectures') if isinstance(config.get('architectures'),list) else []
|
|
architecture=next((x for x in architectures if isinstance(x,str) and 0<len(x)<=100),None)
|
|
architecture=architecture or next((config.get(k) for k in ('_class_name','model_type') if isinstance(config.get(k),str) and 0<len(config[k])<=100),None)
|
|
tags=[x for x in model.get('tags',[]) if isinstance(x,str) and len(x)<=80][:32] if isinstance(model.get('tags'),list) else []
|
|
return dict(pipeline_tag=model.get('pipeline_tag') if isinstance(model.get('pipeline_tag'),str) and len(model['pipeline_tag'])<=80 else None,
|
|
library_name=model.get('library_name') if isinstance(model.get('library_name'),str) and len(model['library_name'])<=80 else None,
|
|
architecture=architecture,tags=tags)
|
|
|
|
class Catalog:
|
|
def __init__(self,root):
|
|
from hub_auth import HubAuth
|
|
self.hub_auth=HubAuth(Path(root)/'huggingface.json')
|
|
self.root=Path(root);self.lock=threading.RLock();self.job=None;self.cancel=threading.Event()
|
|
self.cache={};self.cache_lock=threading.Lock();self.history=[];self.pending=[];self.closing=False;self.samples=[]
|
|
path=self.root/'downloads.json'
|
|
if path.exists():
|
|
self.history=json.loads(path.read_text())
|
|
for job in self.history:
|
|
if job['state'] in ('downloading','queued'):job.update(state='interrupted',error='Deck wurde neu gestartet. Datei erneut auswählen und herunterladen.')
|
|
else:
|
|
for entry in self.root.glob('*/entry.json'):
|
|
try:
|
|
x=json.loads(entry.read_text())
|
|
self.history.append(dict(id='import-'+entry.parent.name,repo=x['repo'],file=x['file'],kind=x['kind'],state='complete',bytes=x['size'],total=x['size'],created_at=x.get('downloaded_at'),error=None))
|
|
except (OSError,ValueError,KeyError):pass
|
|
if self.history:self._save_history()
|
|
def hub_credentials(self,token):
|
|
with self.lock:
|
|
self.hub_auth.save(token)
|
|
with self.cache_lock:self.cache.clear()
|
|
return self.hub_auth.status()
|
|
def _metadata(self,path):
|
|
return metadata(path,self.hub_auth.token())
|
|
def _save_history(self):
|
|
self.root.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
temp=self.root/'downloads.tmp';temp.write_text(json.dumps(self.history));temp.replace(self.root/'downloads.json')
|
|
def dismiss(self,job_id):
|
|
with self.lock:
|
|
job=next((x for x in self.history if x['id']==job_id),None)
|
|
if not job:raise ValueError('Download nicht gefunden.')
|
|
if job['state']=='downloading':raise ValueError('Laufenden Download zuerst abbrechen.')
|
|
if job['state']=='queued':
|
|
self.pending=[x for x in self.pending if x[0]['id']!=job_id]
|
|
job['state']='cancelled'
|
|
job['dismissed']=True
|
|
if self.job and self.job.get('id')==job_id:self.job=None
|
|
self._save_history()
|
|
return {'dismissed':True,'model_deleted':False}
|
|
def entry(self,ident):
|
|
if not isinstance(ident,str) or not re.fullmatch('[a-f0-9]{64}',ident):raise ValueError('Ungültige Modelldatei-ID.')
|
|
entry=next((x for x in self.status()['entries'] if x['id']==ident),None)
|
|
if not entry:raise ValueError('Modelldatei nicht in der Bibliothek.')
|
|
path=self.root/ident/('model'+PurePosixPath(entry['file']).suffix)
|
|
if not path.is_file() or path.stat().st_size!=entry['size']:raise ValueError('Modelldatei fehlt oder ist unvollständig.')
|
|
return entry
|
|
def search(self,q,kind,sort="downloads",base_only=False,purpose="model"):
|
|
if purpose not in ('model','projector'):raise ValueError('Ungültiger Suchzweck.')
|
|
if kind not in KINDS or not isinstance(q,str) or len(q)>120:raise ValueError('Ungültige Suche.')
|
|
if sort not in ('downloads','name','newest'):raise ValueError('Ungültige Sortierung.')
|
|
if not isinstance(base_only,bool):raise ValueError('Ungültiger Basismodellfilter.')
|
|
q=q.strip();direct=None;notice=None
|
|
if q.startswith('https://huggingface.co/'):
|
|
url=urllib.parse.urlsplit(q)
|
|
if url.netloc!='huggingface.co' or url.query or url.fragment:raise ValueError('Bitte einen direkten Hugging-Face-Repository-Link eingeben.')
|
|
direct=repo_id(url.path.strip('/'))
|
|
elif '/' in q:direct=repo_id(q)
|
|
if direct:
|
|
item=self._metadata('/api/models/'+direct)
|
|
return dict(models=[dict(repo=item['id'],downloads=item.get('downloads'),gated=item.get('gated',False),created_at=item.get('createdAt'),capabilities=capabilities(item),runtime_metadata=runtime_metadata(item))],base_only=base_only,scanned=1,notice='Direkt ausgewähltes Repository: Kategorie- und Basismodellfilter werden für diesen Treffer nicht angewendet.')
|
|
if q.lower().endswith('.gguf'):
|
|
q=q[:-5];notice='GGUF-Dateiname als Repository-Suchbegriff verwendet. Die Datei wählst du anschließend in den Modelldetails.'
|
|
query=dict(search=q,filter='gguf' if purpose=='projector' else KINDS[kind],limit=100 if base_only else 20,sort='createdAt' if sort=='newest' else 'downloads',direction=-1)
|
|
query['expand']=['downloads','createdAt','gated','cardData','tags','pipeline_tag','library_name','config']
|
|
rows=self._metadata('/api/models?'+urllib.parse.urlencode(query,doseq=True))
|
|
if not rows and q and kind=='chat' and purpose=='model':
|
|
fallback=dict(query,filter='gguf',expand=['downloads','createdAt','gated','cardData','tags','pipeline_tag'])
|
|
candidates=self._metadata('/api/models?'+urllib.parse.urlencode(fallback,doseq=True))
|
|
rows=[x for x in candidates if x.get('pipeline_tag')=='text-generation' or 'conversational' in (x.get('tags') or [])]
|
|
if rows:notice='GGUF-Sprachmodelle ohne reguläres Kategorie-Tag gefunden. Bei aktiver Basismodellauswahl bleiben Quantisierungen ausgeblendet.'
|
|
scanned=len(rows)
|
|
if base_only:rows=[x for x in rows if not is_derived_model(x)]
|
|
return {'models':[dict(repo=x['id'],downloads=x.get('downloads'),gated=x.get('gated',False),created_at=x.get('createdAt'),capabilities=capabilities(x),runtime_metadata=runtime_metadata(x)) for x in rows[:20]],'base_only':base_only,'scanned':scanned,'notice':notice}
|
|
def files(self,repo,revision=None):
|
|
repo=repo_id(repo)
|
|
if revision is not None and (not isinstance(revision,str) or not re.fullmatch("[a-f0-9]{40}",revision)):raise ValueError("Ungültige Quellversion.")
|
|
cache_key=(repo,revision)
|
|
with self.cache_lock:
|
|
cached=self.cache.get(cache_key)
|
|
if cached and time.monotonic()-cached[0]<300:return cached[1]
|
|
data=self._metadata('/api/models/'+repo+('/revision/'+revision if revision else '')+'?blobs=true')
|
|
revision=data.get('sha','')
|
|
if not re.fullmatch('[a-f0-9]{40}',revision):raise ValueError('Keine feste Repository-Version verfügbar.')
|
|
files=[]
|
|
for f in data.get('siblings',[]):
|
|
name=f['rfilename'];p=PurePosixPath(name)
|
|
if p.is_absolute() or '..' in p.parts or p.suffix.lower() not in ('.gguf','.safetensors','.json'):continue
|
|
size=f.get('size',f.get('lfs',{}).get('size'))
|
|
if not isinstance(size,int) or size<1:continue
|
|
files.append(dict(name=name,size=size,sha256=f.get('lfs',{}).get('sha256')))
|
|
result=dict(auth_configured=self.hub_auth.status()['configured'],capabilities=capabilities(data),runtime_metadata=runtime_metadata(data),repo=repo,revision=revision,files=files,gated=data.get('gated',False),license=(data.get('cardData') or {}).get('license'),url='https://huggingface.co/'+repo)
|
|
with self.cache_lock:
|
|
if len(self.cache)>=64:self.cache.pop(next(iter(self.cache)))
|
|
self.cache[cache_key]=(time.monotonic(),result)
|
|
return result
|
|
def status(self):
|
|
with self.lock:
|
|
entries=[]
|
|
if self.root.exists():
|
|
for p in self.root.glob('*/entry.json'):
|
|
try:
|
|
item=json.loads(p.read_text());item['id']=p.parent.name;item.update(file_role(item['file']));entries.append(item)
|
|
except (OSError,ValueError):pass
|
|
return dict(entries=entries,downloads=[dict(x) for x in reversed(self.history) if not x.get("dismissed")],job=dict(self.job) if self.job else None,free_bytes=shutil.disk_usage(self.root if self.root.exists() else self.root.parent).free)
|
|
def start(self,repo,filename,revision,kind):
|
|
if kind not in KINDS:raise ValueError('Ungültiger Bereich.')
|
|
with self.lock:
|
|
if self.closing:raise ValueError('Deck wird beendet.')
|
|
if len(self.pending)>=20:raise ValueError('Warteschlange voll (20 Dateien).')
|
|
with self.cache_lock:self.cache.pop((repo,revision),None)
|
|
data=self.files(repo,revision)
|
|
if data['gated'] and not self.hub_auth.token():raise ValueError('Dieses Repository benötigt eine Freigabe und einen Hugging-Face-Token unter Einstellungen → Zugang.')
|
|
if revision!=data['revision']:raise ValueError('Repository wurde geändert. Dateiliste neu laden.')
|
|
item=next((x for x in data['files'] if x['name']==filename),None)
|
|
if not item:raise ValueError('Datei nicht verfügbar.')
|
|
self.root.mkdir(parents=True,exist_ok=True,mode=0o700)
|
|
reserved=sum(x['total']-x['bytes'] for x in self.history if x['state'] in ('downloading','queued'))
|
|
if item['size']+reserved>shutil.disk_usage(self.root).free-10*1024**3:raise ValueError('Nicht genug Platz mit 10 GiB freier Reserve.')
|
|
ident=hashlib.sha256((repo+revision+filename).encode()).hexdigest()
|
|
if any(x.get('entry_id')==ident and x['state'] in ('downloading','queued') for x in self.history):raise ValueError('Datei läuft bereits oder steht in der Warteschlange.')
|
|
target=self.root/ident
|
|
if (target/'entry.json').exists():raise ValueError('Datei bereits in der Bibliothek.')
|
|
target.mkdir(exist_ok=True,mode=0o700)
|
|
job=dict(id=uuid.uuid4().hex,entry_id=ident,state='queued',repo=repo,file=filename,kind=kind,revision=revision,created_at=time.time(),bytes=0,total=item['size'],error=None,bytes_per_second=None,eta_seconds=None)
|
|
self.history.append(job);self.pending.append((job,data,item,target,kind));self._next();self._save_history()
|
|
return dict(job)
|
|
def _next(self):
|
|
if self.closing or (self.job and self.job['state']=='downloading') or not self.pending:return
|
|
self.job,data,item,target,kind=self.pending.pop(0)
|
|
self.job['state']='downloading';self.cancel.clear();self.samples=[(time.monotonic(),0)]
|
|
threading.Thread(target=self._download,args=(data,item,target,kind),daemon=True).start()
|
|
def stop(self,shutdown=False):
|
|
with self.lock:
|
|
if shutdown:self.closing=True
|
|
self.cancel.set()
|
|
return {'cancellation_requested':True}
|
|
def _download(self,data,item,target,kind):
|
|
partial=target/'download.part';dest=target/('model'+PurePosixPath(item['name']).suffix)
|
|
try:
|
|
digest=hashlib.sha256();received=0
|
|
url='https://huggingface.co/'+data['repo']+'/resolve/'+data['revision']+'/'+urllib.parse.quote(item['name'],safe='/')
|
|
with remote(url,self.hub_auth.token()) as r, partial.open('wb') as out:
|
|
while True:
|
|
if self.cancel.is_set():raise InterruptedError()
|
|
chunk=r.read(1024*1024)
|
|
if not chunk:break
|
|
received+=len(chunk)
|
|
if received>item['size'] or shutil.disk_usage(target).free<10*1024**3:raise ValueError('Größe oder Speicherreserve überschritten.')
|
|
out.write(chunk);digest.update(chunk)
|
|
with self.lock:
|
|
now=time.monotonic();self.samples.append((now,received))
|
|
while len(self.samples)>2 and self.samples[1][0]<now-10:self.samples.pop(0)
|
|
elapsed=now-self.samples[0][0];speed=(received-self.samples[0][1])/elapsed if elapsed>=.25 else None
|
|
self.job.update(bytes=received,bytes_per_second=speed,eta_seconds=(item['size']-received)/speed if speed else None)
|
|
if received!=item['size']:raise ValueError('Unvollständiger Download.')
|
|
if item['sha256'] and digest.hexdigest()!=item['sha256']:raise ValueError('SHA-256-Prüfung fehlgeschlagen.')
|
|
partial.replace(dest)
|
|
entry=dict(repo=data['repo'],revision=data['revision'],file=item['name'],size=received,sha256=digest.hexdigest(),upstream_hash_verified=bool(item['sha256']),kind=kind,downloaded_at=time.time(),state='downloaded',runtime_ready=False,runtime_metadata=data.get('runtime_metadata',{}))
|
|
temp=target/'entry.tmp';temp.write_text(json.dumps(entry));temp.replace(target/'entry.json')
|
|
terminal='complete';error=None
|
|
except Exception as exc:
|
|
terminal='cancelled' if isinstance(exc,InterruptedError) else 'failed'
|
|
error=str(exc) if isinstance(exc,ValueError) else ('Abgebrochen.' if isinstance(exc,InterruptedError) else 'Download fehlgeschlagen; Verbindung oder Anbieter prüfen.')
|
|
partial.unlink(missing_ok=True)
|
|
finally:
|
|
with self.lock:
|
|
self.job.update(state=terminal,error=error,bytes_per_second=None,eta_seconds=None)
|
|
self._save_history();self._next()
|