Enable public model catalogue and isolated file downloads; define native target

2026-09-28 15:50:31 +02:00
parent 6d5671f75b
commit 5fb732469a
12 changed files with 247 additions and 17 deletions
@@ -0,0 +1,113 @@
+"""Public Hub catalogue and bounded, serial downloads; independent of Docker/systemd."""
+import hashlib
+import json
+import os
+from pathlib import Path, PurePosixPath
+import re
+import shutil
+import threading
+import time
+import urllib.parse
+import urllib.request
+
+KINDS={'chat':'text-generation','image':'text-to-image','audio':'text-to-speech','video':'text-to-video'}
+
+def safe_url(url):
+    p=urllib.parse.urlsplit(url)
+    if p.scheme!='https' or p.username or p.password or p.port not in (None,443) or not any(p.hostname==d or (p.hostname or '').endswith('.'+d) for d in ('huggingface.co','hf.co','xethub.hf.co')):
+        raise ValueError('Downloadziel außerhalb der erlaubten Hugging-Face-Domains.')
+    return url
+
+class Redirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return super().redirect_request(req,fp,code,msg,headers,safe_url(newurl))
+
+def remote(url):
+    return urllib.request.build_opener(Redirect()).open(urllib.request.Request(safe_url(url),headers={'User-Agent':'Athena-Deck/0.5'}),timeout=20)
+
+def metadata(path):
+    with remote('https://huggingface.co'+path) as r:
+        raw=r.read(8*1024*1024+1)
+    if len(raw)>8*1024*1024:raise ValueError('Metadaten zu groß; Repository wird noch nicht unterstützt.')
+    return json.loads(raw)
+
+def repo_id(value):
+    if not isinstance(value,str) or not re.fullmatch(r'[A-Za-z0-9_-][A-Za-z0-9_.-]{0,95}/[A-Za-z0-9_-][A-Za-z0-9_.-]{0,95}',value):raise ValueError('Ungültige Repository-ID.')
+    return value
+
+class Catalog:
+    def __init__(self,root):
+        self.root=Path(root);self.lock=threading.RLock();self.job=None;self.cancel=threading.Event()
+    def search(self,q,kind):
+        if kind not in KINDS or not isinstance(q,str) or len(q)>120:raise ValueError('Ungültige Suche.')
+        rows=metadata('/api/models?'+urllib.parse.urlencode(dict(search=q,filter=KINDS[kind],limit=20,sort='downloads',direction=-1)))
+        return {'models':[dict(repo=x['id'],downloads=x.get('downloads'),gated=x.get('gated',False)) for x in rows]}
+    def files(self,repo):
+        repo=repo_id(repo)
+        data=metadata('/api/models/'+repo+'?blobs=true')
+        revision=data.get('sha','')
+        if not re.fullmatch('[a-f0-9]{40}',revision):raise ValueError('Keine feste Repository-Version verfügbar.')
+        files=[]
+        for f in data.get('siblings',[]):
+            name=f['rfilename'];p=PurePosixPath(name)
+            if p.is_absolute() or '..' in p.parts or p.suffix.lower() not in ('.gguf','.safetensors','.json'):continue
+            size=f.get('size',f.get('lfs',{}).get('size'))
+            if not isinstance(size,int) or size<1:continue
+            files.append(dict(name=name,size=size,sha256=f.get('lfs',{}).get('sha256')))
+        return dict(repo=repo,revision=revision,files=files,gated=data.get('gated',False),license=(data.get('cardData') or {}).get('license'),url='https://huggingface.co/'+repo)
+    def status(self):
+        with self.lock:
+            entries=[]
+            if self.root.exists():
+                for p in self.root.glob('*/entry.json'):
+                    try:
+                        item=json.loads(p.read_text());entries.append(item)
+                    except (OSError,ValueError):pass
+            return dict(entries=entries,job=dict(self.job) if self.job else None,free_bytes=shutil.disk_usage(self.root if self.root.exists() else self.root.parent).free)
+    def start(self,repo,filename,revision,kind):
+        if kind not in KINDS:raise ValueError('Ungültiger Bereich.')
+        with self.lock:
+            if self.job and self.job['state']=='downloading':raise ValueError('Ein Download läuft bereits.')
+            data=self.files(repo)
+            if data['gated']:raise ValueError('Zugangsbeschränkte Modelle werden noch nicht unterstützt.')
+            if revision!=data['revision']:raise ValueError('Repository wurde geändert. Dateiliste neu laden.')
+            item=next((x for x in data['files'] if x['name']==filename),None)
+            if not item:raise ValueError('Datei nicht verfügbar.')
+            self.root.mkdir(parents=True,exist_ok=True,mode=0o700)
+            if item['size']>shutil.disk_usage(self.root).free-10*1024**3:raise ValueError('Nicht genug Platz mit 10 GiB freier Reserve.')
+            ident=hashlib.sha256((repo+revision+filename).encode()).hexdigest()
+            target=self.root/ident
+            if (target/'entry.json').exists():raise ValueError('Datei bereits in der Bibliothek.')
+            target.mkdir(exist_ok=True,mode=0o700)
+            self.cancel.clear()
+            self.job=dict(id=ident,state='downloading',repo=repo,file=filename,bytes=0,total=item['size'],error=None)
+            threading.Thread(target=self._download,args=(data,item,target,kind),daemon=True).start()
+            return dict(self.job)
+    def stop(self):
+        self.cancel.set()
+        return {'cancellation_requested':True}
+    def _download(self,data,item,target,kind):
+        partial=target/'download.part';dest=target/('model'+PurePosixPath(item['name']).suffix)
+        try:
+            digest=hashlib.sha256();received=0
+            url='https://huggingface.co/'+data['repo']+'/resolve/'+data['revision']+'/'+urllib.parse.quote(item['name'],safe='/')
+            with remote(url) as r, partial.open('wb') as out:
+                while True:
+                    if self.cancel.is_set():raise InterruptedError()
+                    chunk=r.read(1024*1024)
+                    if not chunk:break
+                    received+=len(chunk)
+                    if received>item['size'] or shutil.disk_usage(target).free<10*1024**3:raise ValueError('Größe oder Speicherreserve überschritten.')
+                    out.write(chunk);digest.update(chunk)
+                    with self.lock:self.job['bytes']=received
+            if received!=item['size']:raise ValueError('Unvollständiger Download.')
+            if item['sha256'] and digest.hexdigest()!=item['sha256']:raise ValueError('SHA-256-Prüfung fehlgeschlagen.')
+            partial.replace(dest)
+            entry=dict(repo=data['repo'],revision=data['revision'],file=item['name'],size=received,sha256=digest.hexdigest(),upstream_hash_verified=bool(item['sha256']),kind=kind,downloaded_at=time.time(),state='downloaded',runtime_ready=False)
+            temp=target/'entry.tmp';temp.write_text(json.dumps(entry));temp.replace(target/'entry.json')
+            with self.lock:self.job['state']='complete'
+        except Exception as exc:
+            with self.lock:
+                self.job['state']='cancelled' if isinstance(exc,InterruptedError) else 'failed'
+                self.job['error']=str(exc) if isinstance(exc,ValueError) else ('Abgebrochen.' if isinstance(exc,InterruptedError) else 'Download fehlgeschlagen; Verbindung oder Anbieter prüfen.')
+            partial.unlink(missing_ok=True)