This commit is contained in:
jaydrennan 2024-03-14 20:09:24 -07:00
parent f0b7f31ba5
commit 7a2efb3d0c
4 changed files with 137 additions and 52 deletions

View File

@ -97,6 +97,7 @@ def generate_single_image(
weights_config=prompt.model_weights, weights_config=prompt.model_weights,
for_inpainting=prompt.should_use_inpainting for_inpainting=prompt.should_use_inpainting
and prompt.inpaint_method == "finetune", and prompt.inpaint_method == "finetune",
for_ipadapter=prompt.should_use_ipadapter,
dtype=dtype, dtype=dtype,
) )
lc.model = sd lc.model = sd

View File

@ -102,6 +102,7 @@ def _imagine_cmd(
configure_logging(log_level) configure_logging(log_level)
init_images = [init_image] if isinstance(init_image, str) else init_image init_images = [init_image] if isinstance(init_image, str) else init_image
image_prompts = [image_prompt] if isinstance(image_prompt, str) else image_prompt
from imaginairy.utils import glob_expand_paths from imaginairy.utils import glob_expand_paths
@ -112,6 +113,13 @@ def _imagine_cmd(
msg = f"Could not find any images matching the glob pattern(s) {init_image}. Are you sure the file(s) exists?" msg = f"Could not find any images matching the glob pattern(s) {init_image}. Are you sure the file(s) exists?"
raise ValueError(msg) raise ValueError(msg)
num_prexpanded_init_image_prompts = len(image_prompts)
image_prompts = glob_expand_paths(image_prompts)
if len(image_prompts) < num_prexpanded_init_image_prompts:
msg = f"Could not find any images matching the glob pattern(s) {init_image}. Are you sure the file(s) exists?"
raise ValueError(msg)
total_image_count = len(prompt_texts) * max(len(init_images), 1) * repeats total_image_count = len(prompt_texts) * max(len(init_images), 1) * repeats
img_msg = "" img_msg = ""
if len(init_images) > 0: if len(init_images) > 0:
@ -123,27 +131,32 @@ def _imagine_cmd(
from imaginairy.api import imagine_image_files from imaginairy.api import imagine_image_files
from imaginairy.schema import ImaginePrompt, LazyLoadingImage from imaginairy.schema import ImaginePrompt, LazyLoadingImage
new_init_images = [] # new_init_images = []
for _init_image in init_images: # for _init_image in init_images:
if _init_image and _init_image.startswith("http"): # if _init_image and _init_image.startswith("http"):
_init_image = LazyLoadingImage(url=_init_image) # _init_image = LazyLoadingImage(url=_init_image)
elif _init_image.startswith("textimg="): # elif _init_image.startswith("textimg="):
from imaginairy.utils import named_resolutions # from imaginairy.utils import named_resolutions
from imaginairy.utils.text_image import image_from_textimg_str # from imaginairy.utils.text_image import image_from_textimg_str
#
resolved_width, resolved_height = named_resolutions.normalize_image_size( # resolved_width, resolved_height = named_resolutions.normalize_image_size(
size # size
) # )
_init_image = image_from_textimg_str( # _init_image = image_from_textimg_str(
_init_image, resolved_width, resolved_height # _init_image, resolved_width, resolved_height
) # )
else: # else:
_init_image = LazyLoadingImage(filepath=_init_image) # _init_image = LazyLoadingImage(filepath=_init_image)
new_init_images.append(_init_image) # new_init_images.append(_init_image)
init_images = new_init_images # init_images = new_init_images
init_images = images_to_lazyloaders(init_images, size)
if not init_images: if not init_images:
init_images = [None] init_images = [None]
image_prompts = images_to_lazyloaders(image_prompts, size)
if not image_prompts:
image_prompts = [None]
if mask_image: if mask_image:
if mask_image.startswith("http"): if mask_image.startswith("http"):
mask_image = LazyLoadingImage(url=mask_image) mask_image = LazyLoadingImage(url=mask_image)
@ -188,6 +201,46 @@ def _imagine_cmd(
prompt_strength=prompt_strength, prompt_strength=prompt_strength,
init_image=_init_image, init_image=_init_image,
init_image_strength=init_image_strength, init_image_strength=init_image_strength,
# image_prompt=image_prompt,
# image_prompt_strength=image_prompt_strength,
control_inputs=control_inputs,
seed=seed,
solver_type=solver,
steps=steps,
size=size,
mask_image=mask_image,
mask_prompt=mask_prompt,
mask_mode=mask_mode,
mask_modify_original=mask_modify_original,
outpaint=outpaint,
upscale=upscale,
fix_faces=fix_faces,
fix_faces_fidelity=fix_faces_fidelity,
tile_mode=_tile_mode,
allow_compose_phase=allow_compose_phase,
model_weights=model_weights_path,
caption_text=caption_text,
composition_strength=composition_strength,
)
from imaginairy.utils.prompt_schedules import (
parse_schedule_strs,
prompt_mutator,
)
if arg_schedules:
schedules = parse_schedule_strs(arg_schedules)
for new_prompt in prompt_mutator(prompt, schedules):
prompts.append(new_prompt)
else:
prompts.append(prompt)
for image_prompt in image_prompts:
prompt = ImaginePrompt(
prompt=next(prompt_iterator),
negative_prompt=negative_prompt,
prompt_strength=prompt_strength,
# init_image=_init_image,
# init_image_strength=init_image_strength,
image_prompt=image_prompt, image_prompt=image_prompt,
image_prompt_strength=image_prompt_strength, image_prompt_strength=image_prompt_strength,
control_inputs=control_inputs, control_inputs=control_inputs,
@ -256,6 +309,29 @@ def _imagine_cmd(
logger.info(f"[compilation] saved to: {new_filename}") logger.info(f"[compilation] saved to: {new_filename}")
def images_to_lazyloaders(images, size):
from imaginairy.schema import LazyLoadingImage
lazyloaders = []
for image in images:
if image and image.startswith("http"):
image = LazyLoadingImage(url=image)
elif image.startswith("textimg="):
from imaginairy.utils import named_resolutions
from imaginairy.utils.text_image import image_from_textimg_str
resolved_width, resolved_height = named_resolutions.normalize_image_size(
size
)
image = image_from_textimg_str(
image, resolved_width, resolved_height
)
else:
image = LazyLoadingImage(filepath=image)
lazyloaders.append(image)
return lazyloaders
def add_options(options): def add_options(options):
def _add_options(func): def _add_options(func):
for option in reversed(options): for option in reversed(options):
@ -319,7 +395,7 @@ common_options = [
click.option( click.option(
"--image-prompt", "--image-prompt",
metavar="PATH|URL", metavar="PATH|URL",
help="Starting image.", help="Image to be used as part of the image and test prompt.",
multiple=True, multiple=True,
), ),
click.option( click.option(

View File

@ -782,6 +782,10 @@ class ImaginePrompt(BaseModel, protected_namespaces=()):
def should_use_inpainting_weights(self) -> bool: def should_use_inpainting_weights(self) -> bool:
return self.should_use_inpainting and self.inpaint_method == "finetune" return self.should_use_inpainting and self.inpaint_method == "finetune"
@property
def should_use_ipadapter(self) -> bool:
return bool(self.image_prompt)
@property @property
def model_architecture(self) -> config.ModelArchitecture: def model_architecture(self) -> config.ModelArchitecture:
return self.model_weights.architecture return self.model_weights.architecture

View File

@ -191,6 +191,7 @@ def _get_diffusion_model(
def get_diffusion_model_refiners( def get_diffusion_model_refiners(
weights_config: iconfig.ModelWeightsConfig, weights_config: iconfig.ModelWeightsConfig,
for_inpainting=False, for_inpainting=False,
for_ipadapter=False,
dtype=None, dtype=None,
) -> LatentDiffusionModel: ) -> LatentDiffusionModel:
"""Load a diffusion model.""" """Load a diffusion model."""
@ -204,41 +205,44 @@ def get_diffusion_model_refiners(
# ensures a "fresh" copy that doesn't have additional injected parts # ensures a "fresh" copy that doesn't have additional injected parts
sd = sd.structural_copy() sd = sd.structural_copy()
# inject ip-adapter (img to img prompt) if for_ipadapter:
from PIL import Image # inject ip-adapter (img to img prompt)
from PIL import Image
from imaginairy.vendored.refiners.fluxion.utils import ( from imaginairy.vendored.refiners.fluxion.utils import (
load_from_safetensors, load_from_safetensors,
no_grad, no_grad,
)
from imaginairy.vendored.refiners.foundationals.latent_diffusion import (
SDXLIPAdapter,
)
image_prompt = Image.open(
"/imaginAIry/docs/assets/000032_337692011_PLMS40_PS7.5_a_photo_of_a_dog.jpg"
)
ip_adapter = SDXLIPAdapter(
target=sd.unet,
weights=load_from_safetensors(
"/imaginAIry/imaginairy/utils/ip-adapter_sdxl_vit-h.safetensors"
),
)
ip_adapter.clip_image_encoder.load_from_safetensors(
"/imaginAIry/imaginairy/utils/clip_image.safetensors"
)
ip_adapter.inject()
scale = 0.4
ip_adapter.set_scale(scale)
print(f"SCALE: {scale}")
with no_grad():
clip_image_embedding = ip_adapter.compute_clip_image_embedding(
ip_adapter.preprocess_image(image_prompt)
) )
ip_adapter.set_clip_image_embedding(clip_image_embedding) from imaginairy.vendored.refiners.foundationals.latent_diffusion import (
SDXLIPAdapter,
)
image_prompt = Image.open(
"/Users/jaydrennan/projects/imaginAIry/docs/assets/mona-lisa-headshot-anim.gif"
)
ip_adapter = SDXLIPAdapter(
target=sd.unet,
weights=load_from_safetensors(
"/imaginAIry/imaginairy/utils/ip-adapter_sdxl_vit-h.safetensors"
),
)
ip_adapter.clip_image_encoder.load_from_safetensors(
"/imaginAIry/imaginairy/utils/clip_image.safetensors"
)
ip_adapter.inject()
scale = 0.4
ip_adapter.set_scale(scale)
print(f"SCALE: {scale}")
with no_grad():
clip_image_embedding = ip_adapter.compute_clip_image_embedding(
ip_adapter.preprocess_image(image_prompt)
)
ip_adapter.set_clip_image_embedding(clip_image_embedding)
sd.set_self_attention_guidance(enable=True) sd.set_self_attention_guidance(enable=True)