mirror of
https://github.com/brycedrennan/imaginAIry
synced 2024-10-31 03:20:40 +00:00
wip
This commit is contained in:
parent
f0b7f31ba5
commit
7a2efb3d0c
@ -97,6 +97,7 @@ def generate_single_image(
|
|||||||
weights_config=prompt.model_weights,
|
weights_config=prompt.model_weights,
|
||||||
for_inpainting=prompt.should_use_inpainting
|
for_inpainting=prompt.should_use_inpainting
|
||||||
and prompt.inpaint_method == "finetune",
|
and prompt.inpaint_method == "finetune",
|
||||||
|
for_ipadapter=prompt.should_use_ipadapter,
|
||||||
dtype=dtype,
|
dtype=dtype,
|
||||||
)
|
)
|
||||||
lc.model = sd
|
lc.model = sd
|
||||||
|
@ -102,6 +102,7 @@ def _imagine_cmd(
|
|||||||
configure_logging(log_level)
|
configure_logging(log_level)
|
||||||
|
|
||||||
init_images = [init_image] if isinstance(init_image, str) else init_image
|
init_images = [init_image] if isinstance(init_image, str) else init_image
|
||||||
|
image_prompts = [image_prompt] if isinstance(image_prompt, str) else image_prompt
|
||||||
|
|
||||||
from imaginairy.utils import glob_expand_paths
|
from imaginairy.utils import glob_expand_paths
|
||||||
|
|
||||||
@ -112,6 +113,13 @@ def _imagine_cmd(
|
|||||||
msg = f"Could not find any images matching the glob pattern(s) {init_image}. Are you sure the file(s) exists?"
|
msg = f"Could not find any images matching the glob pattern(s) {init_image}. Are you sure the file(s) exists?"
|
||||||
raise ValueError(msg)
|
raise ValueError(msg)
|
||||||
|
|
||||||
|
num_prexpanded_init_image_prompts = len(image_prompts)
|
||||||
|
image_prompts = glob_expand_paths(image_prompts)
|
||||||
|
|
||||||
|
if len(image_prompts) < num_prexpanded_init_image_prompts:
|
||||||
|
msg = f"Could not find any images matching the glob pattern(s) {init_image}. Are you sure the file(s) exists?"
|
||||||
|
raise ValueError(msg)
|
||||||
|
|
||||||
total_image_count = len(prompt_texts) * max(len(init_images), 1) * repeats
|
total_image_count = len(prompt_texts) * max(len(init_images), 1) * repeats
|
||||||
img_msg = ""
|
img_msg = ""
|
||||||
if len(init_images) > 0:
|
if len(init_images) > 0:
|
||||||
@ -123,27 +131,32 @@ def _imagine_cmd(
|
|||||||
from imaginairy.api import imagine_image_files
|
from imaginairy.api import imagine_image_files
|
||||||
from imaginairy.schema import ImaginePrompt, LazyLoadingImage
|
from imaginairy.schema import ImaginePrompt, LazyLoadingImage
|
||||||
|
|
||||||
new_init_images = []
|
# new_init_images = []
|
||||||
for _init_image in init_images:
|
# for _init_image in init_images:
|
||||||
if _init_image and _init_image.startswith("http"):
|
# if _init_image and _init_image.startswith("http"):
|
||||||
_init_image = LazyLoadingImage(url=_init_image)
|
# _init_image = LazyLoadingImage(url=_init_image)
|
||||||
elif _init_image.startswith("textimg="):
|
# elif _init_image.startswith("textimg="):
|
||||||
from imaginairy.utils import named_resolutions
|
# from imaginairy.utils import named_resolutions
|
||||||
from imaginairy.utils.text_image import image_from_textimg_str
|
# from imaginairy.utils.text_image import image_from_textimg_str
|
||||||
|
#
|
||||||
resolved_width, resolved_height = named_resolutions.normalize_image_size(
|
# resolved_width, resolved_height = named_resolutions.normalize_image_size(
|
||||||
size
|
# size
|
||||||
)
|
# )
|
||||||
_init_image = image_from_textimg_str(
|
# _init_image = image_from_textimg_str(
|
||||||
_init_image, resolved_width, resolved_height
|
# _init_image, resolved_width, resolved_height
|
||||||
)
|
# )
|
||||||
else:
|
# else:
|
||||||
_init_image = LazyLoadingImage(filepath=_init_image)
|
# _init_image = LazyLoadingImage(filepath=_init_image)
|
||||||
new_init_images.append(_init_image)
|
# new_init_images.append(_init_image)
|
||||||
init_images = new_init_images
|
# init_images = new_init_images
|
||||||
|
init_images = images_to_lazyloaders(init_images, size)
|
||||||
if not init_images:
|
if not init_images:
|
||||||
init_images = [None]
|
init_images = [None]
|
||||||
|
|
||||||
|
image_prompts = images_to_lazyloaders(image_prompts, size)
|
||||||
|
if not image_prompts:
|
||||||
|
image_prompts = [None]
|
||||||
|
|
||||||
if mask_image:
|
if mask_image:
|
||||||
if mask_image.startswith("http"):
|
if mask_image.startswith("http"):
|
||||||
mask_image = LazyLoadingImage(url=mask_image)
|
mask_image = LazyLoadingImage(url=mask_image)
|
||||||
@ -188,6 +201,46 @@ def _imagine_cmd(
|
|||||||
prompt_strength=prompt_strength,
|
prompt_strength=prompt_strength,
|
||||||
init_image=_init_image,
|
init_image=_init_image,
|
||||||
init_image_strength=init_image_strength,
|
init_image_strength=init_image_strength,
|
||||||
|
# image_prompt=image_prompt,
|
||||||
|
# image_prompt_strength=image_prompt_strength,
|
||||||
|
control_inputs=control_inputs,
|
||||||
|
seed=seed,
|
||||||
|
solver_type=solver,
|
||||||
|
steps=steps,
|
||||||
|
size=size,
|
||||||
|
mask_image=mask_image,
|
||||||
|
mask_prompt=mask_prompt,
|
||||||
|
mask_mode=mask_mode,
|
||||||
|
mask_modify_original=mask_modify_original,
|
||||||
|
outpaint=outpaint,
|
||||||
|
upscale=upscale,
|
||||||
|
fix_faces=fix_faces,
|
||||||
|
fix_faces_fidelity=fix_faces_fidelity,
|
||||||
|
tile_mode=_tile_mode,
|
||||||
|
allow_compose_phase=allow_compose_phase,
|
||||||
|
model_weights=model_weights_path,
|
||||||
|
caption_text=caption_text,
|
||||||
|
composition_strength=composition_strength,
|
||||||
|
)
|
||||||
|
from imaginairy.utils.prompt_schedules import (
|
||||||
|
parse_schedule_strs,
|
||||||
|
prompt_mutator,
|
||||||
|
)
|
||||||
|
|
||||||
|
if arg_schedules:
|
||||||
|
schedules = parse_schedule_strs(arg_schedules)
|
||||||
|
for new_prompt in prompt_mutator(prompt, schedules):
|
||||||
|
prompts.append(new_prompt)
|
||||||
|
else:
|
||||||
|
prompts.append(prompt)
|
||||||
|
|
||||||
|
for image_prompt in image_prompts:
|
||||||
|
prompt = ImaginePrompt(
|
||||||
|
prompt=next(prompt_iterator),
|
||||||
|
negative_prompt=negative_prompt,
|
||||||
|
prompt_strength=prompt_strength,
|
||||||
|
# init_image=_init_image,
|
||||||
|
# init_image_strength=init_image_strength,
|
||||||
image_prompt=image_prompt,
|
image_prompt=image_prompt,
|
||||||
image_prompt_strength=image_prompt_strength,
|
image_prompt_strength=image_prompt_strength,
|
||||||
control_inputs=control_inputs,
|
control_inputs=control_inputs,
|
||||||
@ -256,6 +309,29 @@ def _imagine_cmd(
|
|||||||
logger.info(f"[compilation] saved to: {new_filename}")
|
logger.info(f"[compilation] saved to: {new_filename}")
|
||||||
|
|
||||||
|
|
||||||
|
def images_to_lazyloaders(images, size):
|
||||||
|
from imaginairy.schema import LazyLoadingImage
|
||||||
|
lazyloaders = []
|
||||||
|
|
||||||
|
for image in images:
|
||||||
|
if image and image.startswith("http"):
|
||||||
|
image = LazyLoadingImage(url=image)
|
||||||
|
elif image.startswith("textimg="):
|
||||||
|
from imaginairy.utils import named_resolutions
|
||||||
|
from imaginairy.utils.text_image import image_from_textimg_str
|
||||||
|
|
||||||
|
resolved_width, resolved_height = named_resolutions.normalize_image_size(
|
||||||
|
size
|
||||||
|
)
|
||||||
|
image = image_from_textimg_str(
|
||||||
|
image, resolved_width, resolved_height
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
image = LazyLoadingImage(filepath=image)
|
||||||
|
lazyloaders.append(image)
|
||||||
|
return lazyloaders
|
||||||
|
|
||||||
|
|
||||||
def add_options(options):
|
def add_options(options):
|
||||||
def _add_options(func):
|
def _add_options(func):
|
||||||
for option in reversed(options):
|
for option in reversed(options):
|
||||||
@ -319,7 +395,7 @@ common_options = [
|
|||||||
click.option(
|
click.option(
|
||||||
"--image-prompt",
|
"--image-prompt",
|
||||||
metavar="PATH|URL",
|
metavar="PATH|URL",
|
||||||
help="Starting image.",
|
help="Image to be used as part of the image and test prompt.",
|
||||||
multiple=True,
|
multiple=True,
|
||||||
),
|
),
|
||||||
click.option(
|
click.option(
|
||||||
|
@ -782,6 +782,10 @@ class ImaginePrompt(BaseModel, protected_namespaces=()):
|
|||||||
def should_use_inpainting_weights(self) -> bool:
|
def should_use_inpainting_weights(self) -> bool:
|
||||||
return self.should_use_inpainting and self.inpaint_method == "finetune"
|
return self.should_use_inpainting and self.inpaint_method == "finetune"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def should_use_ipadapter(self) -> bool:
|
||||||
|
return bool(self.image_prompt)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def model_architecture(self) -> config.ModelArchitecture:
|
def model_architecture(self) -> config.ModelArchitecture:
|
||||||
return self.model_weights.architecture
|
return self.model_weights.architecture
|
||||||
|
@ -191,6 +191,7 @@ def _get_diffusion_model(
|
|||||||
def get_diffusion_model_refiners(
|
def get_diffusion_model_refiners(
|
||||||
weights_config: iconfig.ModelWeightsConfig,
|
weights_config: iconfig.ModelWeightsConfig,
|
||||||
for_inpainting=False,
|
for_inpainting=False,
|
||||||
|
for_ipadapter=False,
|
||||||
dtype=None,
|
dtype=None,
|
||||||
) -> LatentDiffusionModel:
|
) -> LatentDiffusionModel:
|
||||||
"""Load a diffusion model."""
|
"""Load a diffusion model."""
|
||||||
@ -204,41 +205,44 @@ def get_diffusion_model_refiners(
|
|||||||
# ensures a "fresh" copy that doesn't have additional injected parts
|
# ensures a "fresh" copy that doesn't have additional injected parts
|
||||||
sd = sd.structural_copy()
|
sd = sd.structural_copy()
|
||||||
|
|
||||||
# inject ip-adapter (img to img prompt)
|
if for_ipadapter:
|
||||||
from PIL import Image
|
# inject ip-adapter (img to img prompt)
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
from imaginairy.vendored.refiners.fluxion.utils import (
|
from imaginairy.vendored.refiners.fluxion.utils import (
|
||||||
load_from_safetensors,
|
load_from_safetensors,
|
||||||
no_grad,
|
no_grad,
|
||||||
)
|
|
||||||
from imaginairy.vendored.refiners.foundationals.latent_diffusion import (
|
|
||||||
SDXLIPAdapter,
|
|
||||||
)
|
|
||||||
|
|
||||||
image_prompt = Image.open(
|
|
||||||
"/imaginAIry/docs/assets/000032_337692011_PLMS40_PS7.5_a_photo_of_a_dog.jpg"
|
|
||||||
)
|
|
||||||
|
|
||||||
ip_adapter = SDXLIPAdapter(
|
|
||||||
target=sd.unet,
|
|
||||||
weights=load_from_safetensors(
|
|
||||||
"/imaginAIry/imaginairy/utils/ip-adapter_sdxl_vit-h.safetensors"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
ip_adapter.clip_image_encoder.load_from_safetensors(
|
|
||||||
"/imaginAIry/imaginairy/utils/clip_image.safetensors"
|
|
||||||
)
|
|
||||||
ip_adapter.inject()
|
|
||||||
|
|
||||||
scale = 0.4
|
|
||||||
ip_adapter.set_scale(scale)
|
|
||||||
print(f"SCALE: {scale}")
|
|
||||||
|
|
||||||
with no_grad():
|
|
||||||
clip_image_embedding = ip_adapter.compute_clip_image_embedding(
|
|
||||||
ip_adapter.preprocess_image(image_prompt)
|
|
||||||
)
|
)
|
||||||
ip_adapter.set_clip_image_embedding(clip_image_embedding)
|
from imaginairy.vendored.refiners.foundationals.latent_diffusion import (
|
||||||
|
SDXLIPAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
image_prompt = Image.open(
|
||||||
|
"/Users/jaydrennan/projects/imaginAIry/docs/assets/mona-lisa-headshot-anim.gif"
|
||||||
|
)
|
||||||
|
|
||||||
|
ip_adapter = SDXLIPAdapter(
|
||||||
|
target=sd.unet,
|
||||||
|
weights=load_from_safetensors(
|
||||||
|
"/imaginAIry/imaginairy/utils/ip-adapter_sdxl_vit-h.safetensors"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
ip_adapter.clip_image_encoder.load_from_safetensors(
|
||||||
|
"/imaginAIry/imaginairy/utils/clip_image.safetensors"
|
||||||
|
)
|
||||||
|
ip_adapter.inject()
|
||||||
|
|
||||||
|
scale = 0.4
|
||||||
|
ip_adapter.set_scale(scale)
|
||||||
|
print(f"SCALE: {scale}")
|
||||||
|
|
||||||
|
with no_grad():
|
||||||
|
clip_image_embedding = ip_adapter.compute_clip_image_embedding(
|
||||||
|
ip_adapter.preprocess_image(image_prompt)
|
||||||
|
)
|
||||||
|
ip_adapter.set_clip_image_embedding(clip_image_embedding)
|
||||||
|
|
||||||
sd.set_self_attention_guidance(enable=True)
|
sd.set_self_attention_guidance(enable=True)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user