michael/Athena-Deck
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support authenticated Hugging Face model and component downloads

Browse Source
This commit is contained in:
Mikei386
2026-09-29 11:47:50 +02:00
parent dad67acad8
commit 7f38286f72
13 changed files with 151 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
catalog.py
+32 -12
View File
@@ -10,6 +10,7 @@ import time
import uuid
import urllib.parse
import urllib.request
import urllib.error
KINDS={'chat':'text-generation','image':'text-to-image','audio':'text-to-speech','stt':'automatic-speech-recognition','music':'text-to-audio','voice':'audio-to-audio','video':'text-to-video'}
@@ -21,13 +22,23 @@ def safe_url(url):
class Redirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return super().redirect_request(req,fp,code,msg,headers,safe_url(newurl))
redirected=super().redirect_request(req,fp,code,msg,headers,safe_url(newurl))
if redirected and urllib.parse.urlsplit(req.full_url).netloc != urllib.parse.urlsplit(newurl).netloc:
redirected.remove_header('Authorization')
return redirected
def remote(url):
return urllib.request.build_opener(Redirect()).open(urllib.request.Request(safe_url(url),headers={'User-Agent':'Athena-Deck/0.5'}),timeout=20)
def remote(url,token=None):
headers={'User-Agent':'Athena-Deck/0.5'}
if token and urllib.parse.urlsplit(safe_url(url)).netloc=='huggingface.co':headers['Authorization']='Bearer '+token
try:
return urllib.request.build_opener(Redirect()).open(urllib.request.Request(safe_url(url),headers=headers),timeout=20)
except urllib.error.HTTPError as exc:
code=exc.code;exc.close()
messages={401:'Hugging Face: Token fehlt oder ist ungültig. Unter Einstellungen → Zugang prüfen.',403:'Hugging Face verweigert den Zugriff. Modellfreigabe auf der Modellkarte beantragen und Leserechte des Tokens prüfen.',404:'Hugging-Face-Repository oder Datei nicht gefunden; bei privaten Repositories Token und Leserechte prüfen.'}
raise ValueError(messages.get(code,'Hugging-Face-Anfrage fehlgeschlagen (HTTP '+str(code)+').')) from None
def metadata(path):
with remote('https://huggingface.co'+path) as r:
def metadata(path,token=None):
with remote('https://huggingface.co'+path,token) as r:
raw=r.read(8*1024*1024+1)
if len(raw)>8*1024*1024:raise ValueError('Metadaten zu groß; Repository wird noch nicht unterstützt.')
return json.loads(raw)
@@ -71,6 +82,8 @@ def capabilities(model):
class Catalog:
def __init__(self,root):
from hub_auth import HubAuth
self.hub_auth=HubAuth(Path(root)/'huggingface.json')
self.root=Path(root);self.lock=threading.RLock();self.job=None;self.cancel=threading.Event()
self.cache={};self.cache_lock=threading.Lock();self.history=[];self.pending=[];self.closing=False;self.samples=[]
path=self.root/'downloads.json'
@@ -85,6 +98,13 @@ class Catalog:
self.history.append(dict(id='import-'+entry.parent.name,repo=x['repo'],file=x['file'],kind=x['kind'],state='complete',bytes=x['size'],total=x['size'],created_at=x.get('downloaded_at'),error=None))
except (OSError,ValueError,KeyError):pass
if self.history:self._save_history()
def hub_credentials(self,token):
with self.lock:
self.hub_auth.save(token)
with self.cache_lock:self.cache.clear()
return self.hub_auth.status()
def _metadata(self,path):
return metadata(path,self.hub_auth.token())
def _save_history(self):
self.root.mkdir(parents=True,exist_ok=True,mode=0o700)
temp=self.root/'downloads.tmp';temp.write_text(json.dumps(self.history));temp.replace(self.root/'downloads.json')
@@ -119,16 +139,16 @@ class Catalog:
direct=repo_id(url.path.strip('/'))
elif '/' in q:direct=repo_id(q)
if direct:
item=metadata('/api/models/'+direct)
item=self._metadata('/api/models/'+direct)
return dict(models=[dict(repo=item['id'],downloads=item.get('downloads'),gated=item.get('gated',False),created_at=item.get('createdAt'),capabilities=capabilities(item))],base_only=base_only,scanned=1,notice='Direkt ausgewähltes Repository: Kategorie- und Basismodellfilter werden für diesen Treffer nicht angewendet.')
if q.lower().endswith('.gguf'):
q=q[:-5];notice='GGUF-Dateiname als Repository-Suchbegriff verwendet. Die Datei wählst du anschließend in den Modelldetails.'
query=dict(search=q,filter='gguf' if purpose=='projector' else KINDS[kind],limit=100 if base_only else 20,sort='createdAt' if sort=='newest' else 'downloads',direction=-1)
query['expand']=['downloads','createdAt','gated','cardData','tags','pipeline_tag']
rows=metadata('/api/models?'+urllib.parse.urlencode(query,doseq=True))
rows=self._metadata('/api/models?'+urllib.parse.urlencode(query,doseq=True))
if not rows and q and kind=='chat' and purpose=='model':
fallback=dict(query,filter='gguf',expand=['downloads','createdAt','gated','cardData','tags','pipeline_tag'])
candidates=metadata('/api/models?'+urllib.parse.urlencode(fallback,doseq=True))
candidates=self._metadata('/api/models?'+urllib.parse.urlencode(fallback,doseq=True))
rows=[x for x in candidates if x.get('pipeline_tag')=='text-generation' or 'conversational' in (x.get('tags') or [])]
if rows:notice='GGUF-Sprachmodelle ohne reguläres Kategorie-Tag gefunden. Bei aktiver Basismodellauswahl bleiben Quantisierungen ausgeblendet.'
scanned=len(rows)
@@ -139,7 +159,7 @@ class Catalog:
with self.cache_lock:
cached=self.cache.get(repo)
if cached and time.monotonic()-cached[0]<300:return cached[1]
data=metadata('/api/models/'+repo+'?blobs=true')
data=self._metadata('/api/models/'+repo+'?blobs=true')
revision=data.get('sha','')
if not re.fullmatch('[a-f0-9]{40}',revision):raise ValueError('Keine feste Repository-Version verfügbar.')
files=[]
@@ -149,7 +169,7 @@ class Catalog:
size=f.get('size',f.get('lfs',{}).get('size'))
if not isinstance(size,int) or size<1:continue
files.append(dict(name=name,size=size,sha256=f.get('lfs',{}).get('sha256')))
result=dict(capabilities=capabilities(data),repo=repo,revision=revision,files=files,gated=data.get('gated',False),license=(data.get('cardData') or {}).get('license'),url='https://huggingface.co/'+repo)
result=dict(auth_configured=self.hub_auth.status()['configured'],capabilities=capabilities(data),repo=repo,revision=revision,files=files,gated=data.get('gated',False),license=(data.get('cardData') or {}).get('license'),url='https://huggingface.co/'+repo)
with self.cache_lock:
if len(self.cache)>=64:self.cache.pop(next(iter(self.cache)))
self.cache[repo]=(time.monotonic(),result)
@@ -170,7 +190,7 @@ class Catalog:
if len(self.pending)>=20:raise ValueError('Warteschlange voll (20 Dateien).')
with self.cache_lock:self.cache.pop(repo,None)
data=self.files(repo)
if data['gated']:raise ValueError('Zugangsbeschränkte Modelle werden noch nicht unterstützt.')
if data['gated'] and not self.hub_auth.token():raise ValueError('Dieses Repository benötigt eine Freigabe und einen Hugging-Face-Token unter Einstellungen → Zugang.')
if revision!=data['revision']:raise ValueError('Repository wurde geändert. Dateiliste neu laden.')
item=next((x for x in data['files'] if x['name']==filename),None)
if not item:raise ValueError('Datei nicht verfügbar.')
@@ -200,7 +220,7 @@ class Catalog:
try:
digest=hashlib.sha256();received=0
url='https://huggingface.co/'+data['repo']+'/resolve/'+data['revision']+'/'+urllib.parse.quote(item['name'],safe='/')
with remote(url) as r, partial.open('wb') as out:
with remote(url,self.hub_auth.token()) as r, partial.open('wb') as out:
while True:
if self.cancel.is_set():raise InterruptedError()
chunk=r.read(1024*1024)