From 508a3f41765465476343016f0cc37dbd31783a81 Mon Sep 17 00:00:00 2001 From: Mikei386 <44135113+Mikei386@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:47:36 +0200 Subject: [PATCH] Support FLUX reference images through real edit workflow --- README.md | 2 ++ endpoint.py | 3 ++- image_test.py | 19 +++++++++++++++++-- profiles.py | 4 ++-- test_image_test.py | 17 ++++++++++++++++- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 10a2c46..7f57dee 100644 --- a/README.md +++ b/README.md @@ -230,3 +230,5 @@ Weitere Dienste und nutzt Decks Musikworker statt einer eigenen GPU-Laufzeit. Installation, Verbindung und Einschränkungen: [LadyPoly](deploy/ladypoly/README.md). Sprachmodell-GPUs und Vision-Projektor werden unabhängig zugeordnet. Der Modell-Split verteilt ausschließlich LLM-Gewichte; eine zusätzliche Projektor-GPU wird nur für mmproj sichtbar gemacht, nicht für Modell-Offload. Die Speicherprüfung berücksichtigt beide Geräte einschließlich Projektorbedarf. + +Referenzbilder: Alle derzeit ausführbaren Bildrezepte (Qwen Image 2.1 und FLUX.2 Klein 9B) erlauben bis zu vier Bilder pro Bearbeitungsauftrag. FLUX nutzt VAEEncode und verkettete ReferenceLatent-Nodes für beide Conditioning-Zweige nach dem offiziellen ComfyUI-Workflow: https://github.com/Comfy-Org/workflow_templates/blob/main/templates/image_flux2_klein_image_edit_9b_distilled.json . Neue Modellfamilien benötigen einen passenden Workflow; hochgeladene Referenzen werden niemals stillschweigend verworfen. diff --git a/endpoint.py b/endpoint.py index 2ecc9c4..eea79d6 100644 --- a/endpoint.py +++ b/endpoint.py @@ -348,7 +348,8 @@ class APIHandler(BaseHTTPRequestHandler): if data.get('n',1)!=1 or data.get('response_format','b64_json')!='b64_json':raise APIError('Unterstützt werden n=1 und response_format=b64_json.') profile=ep.find_profile(data.get('model'),'image');params=profile['parameters'] from profiles import image_capabilities - if len(reference_images or [])>image_capabilities(profile.get('model'))['reference_images']:raise APIError('Das aktive Bildprofil unterstützt diese Anzahl Referenzbilder nicht.') + limit=image_capabilities(profile.get('model'))['reference_images'] + if len(reference_images or [])>limit:raise APIError(f"Bildprofil {profile['name']}: {len(reference_images or [])} Referenzbilder erhalten; die Deck-Laufzeitanbindung erlaubt maximal {limit}.") size=data.get('size') if size is not None and (not isinstance(size,str) or (size!='auto' and not re.fullmatch(r'[1-9][0-9]{1,4}x[1-9][0-9]{1,4}',size))):raise APIError('size muss auto oder eine Auflösung wie 1024x1024 sein.') # The selected profile owns resource limits; client size is only a preference. diff --git a/image_test.py b/image_test.py index f2c1f4c..82758ca 100644 --- a/image_test.py +++ b/image_test.py @@ -46,9 +46,10 @@ def select_gpus(devices,model_size,encoder_size,vae_size,offload=False): return max(images,key=lambda g:g['free_mib']),encoders[0] def workflow(prompt,params,seed,family='qwen',references=()): - if references and family!='qwen':raise ValueError('Dieses Bildprofil unterstützt keine Referenzbilder.') + if family not in ('qwen','flux'):raise ValueError('Für diese Modellfamilie fehlt eine Bildlaufzeit-Anbindung.') + if len(references)>4:raise ValueError('Maximal vier Referenzbilder pro Auftrag.') if family=='flux': - return { + graph={ '1':{'class_type':'UNETLoader','inputs':{'unet_name':'model.safetensors','weight_dtype':'default'}}, '2':{'class_type':'DeckTextEncoderLoader','inputs':{'clip_name':'encoder.safetensors','family':'flux2'}}, '3':{'class_type':'VAELoader','inputs':{'vae_name':'vae.safetensors'}}, @@ -64,6 +65,20 @@ def workflow(prompt,params,seed,family='qwen',references=()): '13':{'class_type':'ConditioningZeroOut','inputs':{'conditioning':['4',0]}} } + # FLUX.2 editing uses reference latents on both conditioning branches, + # with fresh output noise; never silently discard uploaded references. + positive=['4',0];negative=['13',0] + for index,name in enumerate(references): + load,scale,encode,pos,neg=map(str,range(20+index*5,25+index*5)) + graph[load]={'class_type':'LoadImage','inputs':{'image':name}} + graph[scale]={'class_type':'ImageScaleToTotalPixels','inputs':{'image':[load,0],'upscale_method':'lanczos','megapixels':1.0,'resolution_steps':1}} + graph[encode]={'class_type':'VAEEncode','inputs':{'pixels':[scale,0],'vae':['3',0]}} + graph[pos]={'class_type':'ReferenceLatent','inputs':{'conditioning':positive,'latent':[encode,0]}} + graph[neg]={'class_type':'ReferenceLatent','inputs':{'conditioning':negative,'latent':[encode,0]}} + positive=[pos,0];negative=[neg,0] + graph['10']['inputs'].update(positive=positive,negative=negative) + return graph + graph={ '1':{'class_type':'UnetLoaderGGUF','inputs':{'unet_name':'model.gguf'}}, '2':{'class_type':'DeckTextEncoderLoader','inputs':{'clip_name':'encoder.safetensors'}}, diff --git a/profiles.py b/profiles.py index 9a97de6..af1c329 100644 --- a/profiles.py +++ b/profiles.py @@ -65,8 +65,8 @@ def image_recipe(model): return None def image_capabilities(model): - """Only advertise reference editing for the verified Qwen Image 2.1 workflow.""" - return {'reference_images':4 if model and model.get('repo')==QWEN_REPO and model.get('file','').endswith('.gguf') else 0} + """All currently supported image recipes have a reference-image workflow.""" + return {'reference_images':4 if model and image_recipe(model) else 0} LTX_REPO='Lightricks/LTX-2.5' LTX_FILE='diffusion_models/ltx-2.5-22b-distilled-transformer-bf16.safetensors' diff --git a/test_image_test.py b/test_image_test.py index 3b93e04..4089243 100644 --- a/test_image_test.py +++ b/test_image_test.py @@ -23,7 +23,22 @@ class ImageTestTests(unittest.TestCase): self.assertEqual(graph['4']['inputs']['image_2'],['22',0]) self.assertEqual(graph['6']['inputs']['latent_image'],['4',2]) self.assertEqual(graph['21']['inputs']['image'],'reference-1.png') - with self.assertRaises(ValueError):workflow('test',p,1,'flux',['reference-1.png']) + with self.assertRaises(ValueError):workflow('test',p,1,'unknown',['reference-1.png']) + def test_flux_references_feed_both_conditioning_branches(self): + p=dict(width=512,height=512,steps=4,guidance=1) + for count in (1,4): + graph=workflow('synthetic edit',p,42,'flux',[f'reference-{i}.png' for i in range(count)]) + self.assertEqual(sum(n['class_type']=='ReferenceLatent' for n in graph.values()),count*2) + self.assertEqual(graph['10']['inputs']['positive'],[str(23+(count-1)*5),0]) + self.assertEqual(graph['10']['inputs']['negative'],[str(24+(count-1)*5),0]) + self.assertEqual(graph['22']['inputs']['vae'],['3',0]) + self.assertEqual(graph['6']['inputs']['latent_image'],['5',0]) + with self.assertRaises(ValueError):workflow('test',p,1,'flux',['r.png']*5) + def test_reference_capabilities_follow_supported_recipes(self): + from profiles import image_capabilities,FLUX_REPO,QWEN_REPO + self.assertEqual(image_capabilities(dict(repo=FLUX_REPO,file='Flux.2 Klein-9B_fp16_nsfw.safetensors'))['reference_images'],4) + self.assertEqual(image_capabilities(dict(repo=QWEN_REPO,file='model.gguf'))['reference_images'],4) + self.assertEqual(image_capabilities(dict(repo='unknown',file='model.safetensors'))['reference_images'],0) def test_reference_validation_enforces_profile_limit_and_signature(self): import base64 png='data:image/png;base64,'+base64.b64encode(b'\x89PNG\r\n\x1a\nsynthetic').decode()