2022-09-09 04:51:25 +00:00
|
|
|
import logging
|
2023-02-12 02:23:45 +00:00
|
|
|
import math
|
2022-09-08 03:59:30 +00:00
|
|
|
import os
|
|
|
|
import re
|
|
|
|
|
2023-02-03 05:43:04 +00:00
|
|
|
from imaginairy.schema import SafetyMode
|
2022-09-10 07:32:31 +00:00
|
|
|
|
2022-09-09 04:51:25 +00:00
|
|
|
logger = logging.getLogger(__name__)
|
2022-09-08 03:59:30 +00:00
|
|
|
|
2022-09-15 02:40:50 +00:00
|
|
|
# leave undocumented. I'd ask that no one publicize this flag. Just want a
|
|
|
|
# slight barrier to entry. Please don't use this is any way that's gonna cause
|
2022-10-10 08:22:11 +00:00
|
|
|
# the media or politicians to freak out about AI...
|
|
|
|
IMAGINAIRY_SAFETY_MODE = os.getenv("IMAGINAIRY_SAFETY_MODE", SafetyMode.STRICT)
|
|
|
|
if IMAGINAIRY_SAFETY_MODE in {"disabled", "classify"}:
|
|
|
|
IMAGINAIRY_SAFETY_MODE = SafetyMode.RELAXED
|
|
|
|
elif IMAGINAIRY_SAFETY_MODE == "filter":
|
|
|
|
IMAGINAIRY_SAFETY_MODE = SafetyMode.STRICT
|
2022-09-11 07:35:57 +00:00
|
|
|
|
2023-01-25 16:55:05 +00:00
|
|
|
# we put this in the global scope so it can be used in the interactive shell
|
|
|
|
_most_recent_result = None
|
|
|
|
|
2022-09-09 04:51:25 +00:00
|
|
|
|
2022-09-10 05:14:04 +00:00
|
|
|
def imagine_image_files(
|
2022-09-08 03:59:30 +00:00
|
|
|
prompts,
|
2022-09-10 05:14:04 +00:00
|
|
|
outdir,
|
2022-09-08 03:59:30 +00:00
|
|
|
precision="autocast",
|
2022-09-11 06:27:22 +00:00
|
|
|
record_step_images=False,
|
|
|
|
output_file_extension="jpg",
|
2022-09-20 04:15:38 +00:00
|
|
|
print_caption=False,
|
2023-01-28 01:18:42 +00:00
|
|
|
make_gif=False,
|
2023-01-29 01:16:47 +00:00
|
|
|
make_compare_gif=False,
|
2023-01-22 01:36:47 +00:00
|
|
|
return_filename_type="generated",
|
2022-09-08 03:59:30 +00:00
|
|
|
):
|
2023-02-03 05:43:04 +00:00
|
|
|
from PIL import ImageDraw
|
|
|
|
|
|
|
|
from imaginairy.animations import make_bounce_animation
|
|
|
|
from imaginairy.img_utils import pillow_fit_image_within
|
|
|
|
|
2022-09-24 18:21:53 +00:00
|
|
|
generated_imgs_path = os.path.join(outdir, "generated")
|
|
|
|
os.makedirs(generated_imgs_path, exist_ok=True)
|
2022-09-13 07:27:53 +00:00
|
|
|
|
2022-09-24 21:41:25 +00:00
|
|
|
base_count = len(os.listdir(generated_imgs_path))
|
2022-09-11 06:27:22 +00:00
|
|
|
output_file_extension = output_file_extension.lower()
|
|
|
|
if output_file_extension not in {"jpg", "png"}:
|
|
|
|
raise ValueError("Must output a png or jpg")
|
2022-09-10 05:14:04 +00:00
|
|
|
|
2022-11-13 03:24:03 +00:00
|
|
|
def _record_step(img, description, image_count, step_count, prompt):
|
2022-09-10 05:14:04 +00:00
|
|
|
steps_path = os.path.join(outdir, "steps", f"{base_count:08}_S{prompt.seed}")
|
|
|
|
os.makedirs(steps_path, exist_ok=True)
|
2022-11-13 03:24:03 +00:00
|
|
|
filename = f"{base_count:08}_S{prompt.seed}_{image_count:04}_step{step_count:03}_{prompt_normalized(description)[:40]}.jpg"
|
2022-09-20 04:15:38 +00:00
|
|
|
|
2022-09-14 07:40:25 +00:00
|
|
|
destination = os.path.join(steps_path, filename)
|
|
|
|
draw = ImageDraw.Draw(img)
|
|
|
|
draw.text((10, 10), str(description))
|
|
|
|
img.save(destination)
|
|
|
|
|
2023-01-28 01:18:42 +00:00
|
|
|
if make_gif:
|
|
|
|
for p in prompts:
|
|
|
|
p.collect_progress_latents = True
|
2023-01-22 01:36:47 +00:00
|
|
|
result_filenames = []
|
2022-09-13 07:27:53 +00:00
|
|
|
for result in imagine(
|
2022-09-10 05:14:04 +00:00
|
|
|
prompts,
|
|
|
|
precision=precision,
|
2022-11-14 06:51:23 +00:00
|
|
|
debug_img_callback=_record_step if record_step_images else None,
|
2022-09-20 04:15:38 +00:00
|
|
|
add_caption=print_caption,
|
2022-09-10 05:14:04 +00:00
|
|
|
):
|
|
|
|
prompt = result.prompt
|
2023-01-25 16:55:05 +00:00
|
|
|
if prompt.is_intermediate:
|
|
|
|
# we don't save intermediate images
|
|
|
|
continue
|
2022-09-28 00:04:16 +00:00
|
|
|
img_str = ""
|
|
|
|
if prompt.init_image:
|
|
|
|
img_str = f"_img2img-{prompt.init_image_strength}"
|
2022-10-22 09:13:06 +00:00
|
|
|
basefilename = (
|
|
|
|
f"{base_count:06}_{prompt.seed}_{prompt.sampler_type.replace('_', '')}{prompt.steps}_"
|
|
|
|
f"PS{prompt.prompt_strength}{img_str}_{prompt_normalized(prompt.prompt_text)}"
|
|
|
|
)
|
2022-09-25 20:07:27 +00:00
|
|
|
|
2022-09-26 04:55:25 +00:00
|
|
|
for image_type in result.images:
|
2022-09-25 20:07:27 +00:00
|
|
|
subpath = os.path.join(outdir, image_type)
|
|
|
|
os.makedirs(subpath, exist_ok=True)
|
|
|
|
filepath = os.path.join(
|
|
|
|
subpath, f"{basefilename}_[{image_type}].{output_file_extension}"
|
|
|
|
)
|
2022-09-26 04:55:25 +00:00
|
|
|
result.save(filepath, image_type=image_type)
|
2023-01-26 04:58:28 +00:00
|
|
|
logger.info(f" [{image_type}] saved to: {filepath}")
|
2023-01-22 01:36:47 +00:00
|
|
|
if image_type == return_filename_type:
|
|
|
|
result_filenames.append(filepath)
|
2023-01-28 01:18:42 +00:00
|
|
|
|
|
|
|
if make_gif and result.progress_latents:
|
2023-01-22 01:36:47 +00:00
|
|
|
subpath = os.path.join(outdir, "gif")
|
|
|
|
os.makedirs(subpath, exist_ok=True)
|
|
|
|
filepath = os.path.join(subpath, f"{basefilename}.gif")
|
2023-01-28 01:18:42 +00:00
|
|
|
|
2023-01-29 01:16:47 +00:00
|
|
|
frames = result.progress_latents + [result.images["generated"]]
|
|
|
|
|
2023-01-28 01:18:42 +00:00
|
|
|
if prompt.init_image:
|
|
|
|
resized_init_image = pillow_fit_image_within(
|
|
|
|
prompt.init_image, prompt.width, prompt.height
|
|
|
|
)
|
|
|
|
frames = [resized_init_image] + frames
|
2023-01-29 05:32:56 +00:00
|
|
|
frames.reverse()
|
|
|
|
make_bounce_animation(
|
|
|
|
imgs=frames,
|
|
|
|
outpath=filepath,
|
|
|
|
start_pause_duration_ms=1500,
|
|
|
|
end_pause_duration_ms=1000,
|
|
|
|
)
|
2023-01-29 01:16:47 +00:00
|
|
|
logger.info(f" [gif] {len(frames)} frames saved to: {filepath}")
|
|
|
|
if make_compare_gif and prompt.init_image:
|
|
|
|
subpath = os.path.join(outdir, "gif")
|
|
|
|
os.makedirs(subpath, exist_ok=True)
|
|
|
|
filepath = os.path.join(subpath, f"{basefilename}_[compare].gif")
|
|
|
|
resized_init_image = pillow_fit_image_within(
|
|
|
|
prompt.init_image, prompt.width, prompt.height
|
|
|
|
)
|
2023-01-29 05:32:56 +00:00
|
|
|
frames = [result.images["generated"], resized_init_image]
|
2023-01-29 01:16:47 +00:00
|
|
|
|
|
|
|
make_bounce_animation(
|
2023-01-28 01:18:42 +00:00
|
|
|
imgs=frames,
|
2023-01-29 01:16:47 +00:00
|
|
|
outpath=filepath,
|
2023-01-22 01:36:47 +00:00
|
|
|
)
|
2023-01-29 01:16:47 +00:00
|
|
|
logger.info(f" [gif-comparison] saved to: {filepath}")
|
|
|
|
|
2022-09-10 05:14:04 +00:00
|
|
|
base_count += 1
|
2022-09-17 05:21:20 +00:00
|
|
|
del result
|
2022-09-10 05:14:04 +00:00
|
|
|
|
2023-01-22 01:36:47 +00:00
|
|
|
return result_filenames
|
|
|
|
|
2022-09-10 05:14:04 +00:00
|
|
|
|
2022-09-13 07:27:53 +00:00
|
|
|
def imagine(
|
2022-09-10 05:14:04 +00:00
|
|
|
prompts,
|
|
|
|
precision="autocast",
|
2022-11-14 06:51:23 +00:00
|
|
|
debug_img_callback=None,
|
|
|
|
progress_img_callback=None,
|
|
|
|
progress_img_interval_steps=3,
|
|
|
|
progress_img_interval_min_s=0.1,
|
2022-09-12 04:32:11 +00:00
|
|
|
half_mode=None,
|
2022-09-20 04:15:38 +00:00
|
|
|
add_caption=False,
|
2023-01-26 04:58:28 +00:00
|
|
|
unsafe_retry_count=1,
|
2022-09-10 05:14:04 +00:00
|
|
|
):
|
2023-02-03 05:43:04 +00:00
|
|
|
import torch.nn
|
|
|
|
|
|
|
|
from imaginairy.schema import ImaginePrompt
|
|
|
|
from imaginairy.utils import (
|
|
|
|
fix_torch_group_norm,
|
|
|
|
fix_torch_nn_layer_norm,
|
|
|
|
get_device,
|
|
|
|
platform_appropriate_autocast,
|
|
|
|
)
|
|
|
|
|
2022-09-10 05:14:04 +00:00
|
|
|
prompts = [ImaginePrompt(prompts)] if isinstance(prompts, str) else prompts
|
|
|
|
prompts = [prompts] if isinstance(prompts, ImaginePrompt) else prompts
|
2022-10-13 05:32:17 +00:00
|
|
|
|
2022-10-24 05:42:17 +00:00
|
|
|
try:
|
|
|
|
num_prompts = str(len(prompts))
|
|
|
|
except TypeError:
|
|
|
|
num_prompts = "?"
|
|
|
|
|
2022-09-22 05:03:12 +00:00
|
|
|
if get_device() == "cpu":
|
|
|
|
logger.info("Running in CPU mode. it's gonna be slooooooow.")
|
2022-09-22 05:38:44 +00:00
|
|
|
|
|
|
|
with torch.no_grad(), platform_appropriate_autocast(
|
|
|
|
precision
|
2022-09-22 05:03:12 +00:00
|
|
|
), fix_torch_nn_layer_norm(), fix_torch_group_norm():
|
2022-10-24 05:42:17 +00:00
|
|
|
for i, prompt in enumerate(prompts):
|
|
|
|
logger.info(
|
|
|
|
f"Generating 🖼 {i + 1}/{num_prompts}: {prompt.prompt_description()}"
|
|
|
|
)
|
2023-01-26 04:58:28 +00:00
|
|
|
for attempt in range(0, unsafe_retry_count + 1):
|
|
|
|
if attempt > 0:
|
|
|
|
prompt.seed += 100_000_000 + attempt
|
|
|
|
result = _generate_single_image(
|
|
|
|
prompt,
|
|
|
|
debug_img_callback=debug_img_callback,
|
|
|
|
progress_img_callback=progress_img_callback,
|
|
|
|
progress_img_interval_steps=progress_img_interval_steps,
|
|
|
|
progress_img_interval_min_s=progress_img_interval_min_s,
|
|
|
|
half_mode=half_mode,
|
|
|
|
add_caption=add_caption,
|
|
|
|
)
|
2023-02-15 16:02:58 +00:00
|
|
|
if not result.safety_score.is_filtered:
|
2023-01-26 04:58:28 +00:00
|
|
|
break
|
|
|
|
if attempt < unsafe_retry_count:
|
2023-02-05 15:43:53 +00:00
|
|
|
logger.info(" Image was unsafe, retrying with new seed...")
|
2023-01-26 04:58:28 +00:00
|
|
|
|
|
|
|
yield result
|
|
|
|
|
|
|
|
|
|
|
|
def _generate_single_image(
|
|
|
|
prompt,
|
|
|
|
debug_img_callback=None,
|
|
|
|
progress_img_callback=None,
|
|
|
|
progress_img_interval_steps=3,
|
|
|
|
progress_img_interval_min_s=0.1,
|
|
|
|
half_mode=None,
|
|
|
|
add_caption=False,
|
2023-02-12 02:23:45 +00:00
|
|
|
suppress_inpaint=False,
|
|
|
|
return_latent=False,
|
2023-01-26 04:58:28 +00:00
|
|
|
):
|
2023-02-03 05:43:04 +00:00
|
|
|
import torch.nn
|
|
|
|
from PIL import Image, ImageOps
|
|
|
|
from pytorch_lightning import seed_everything
|
|
|
|
|
|
|
|
from imaginairy.enhancers.clip_masking import get_img_mask
|
|
|
|
from imaginairy.enhancers.describe_image_blip import generate_caption
|
|
|
|
from imaginairy.enhancers.face_restoration_codeformer import enhance_faces
|
|
|
|
from imaginairy.enhancers.upscale_realesrgan import upscale_image
|
2023-02-15 16:02:36 +00:00
|
|
|
from imaginairy.img_utils import (
|
2023-02-25 08:14:12 +00:00
|
|
|
add_caption_to_image,
|
2023-02-15 16:02:36 +00:00
|
|
|
pillow_fit_image_within,
|
|
|
|
pillow_img_to_torch_image,
|
|
|
|
pillow_mask_to_latent_mask,
|
|
|
|
torch_img_to_pillow_img,
|
|
|
|
)
|
2023-02-03 05:43:04 +00:00
|
|
|
from imaginairy.log_utils import (
|
|
|
|
ImageLoggingContext,
|
|
|
|
log_conditioning,
|
|
|
|
log_img,
|
|
|
|
log_latent,
|
|
|
|
)
|
|
|
|
from imaginairy.model_manager import get_diffusion_model
|
2023-02-15 16:02:36 +00:00
|
|
|
from imaginairy.modules.midas.api import torch_image_to_depth_map
|
2023-02-03 05:43:04 +00:00
|
|
|
from imaginairy.outpaint import outpaint_arg_str_parse, prepare_image_for_outpaint
|
|
|
|
from imaginairy.safety import create_safety_score
|
|
|
|
from imaginairy.samplers import SAMPLER_LOOKUP
|
|
|
|
from imaginairy.samplers.editing import CFGEditingDenoiser
|
|
|
|
from imaginairy.schema import ImaginePrompt, ImagineResult
|
|
|
|
from imaginairy.utils import get_device, randn_seeded
|
|
|
|
|
2023-01-26 04:58:28 +00:00
|
|
|
latent_channels = 4
|
|
|
|
downsampling_factor = 8
|
|
|
|
batch_size = 1
|
|
|
|
global _most_recent_result # noqa
|
|
|
|
# handle prompt pulling in previous values
|
|
|
|
if isinstance(prompt.init_image, str) and prompt.init_image.startswith("*prev"):
|
|
|
|
_, img_type = prompt.init_image.strip("*").split(".")
|
|
|
|
prompt.init_image = _most_recent_result.images[img_type]
|
|
|
|
if isinstance(prompt.mask_image, str) and prompt.mask_image.startswith("*prev"):
|
|
|
|
_, img_type = prompt.mask_image.strip("*").split(".")
|
|
|
|
prompt.mask_image = _most_recent_result.images[img_type]
|
|
|
|
|
|
|
|
model = get_diffusion_model(
|
|
|
|
weights_location=prompt.model,
|
|
|
|
config_path=prompt.model_config_path,
|
2023-02-12 07:42:19 +00:00
|
|
|
control_weights_location=prompt.control_mode,
|
2023-01-26 04:58:28 +00:00
|
|
|
half_mode=half_mode,
|
2023-02-12 02:23:45 +00:00
|
|
|
for_inpainting=(prompt.mask_image or prompt.mask_prompt or prompt.outpaint)
|
|
|
|
and not suppress_inpaint,
|
2023-01-26 04:58:28 +00:00
|
|
|
)
|
2023-02-12 07:42:19 +00:00
|
|
|
is_controlnet_model = hasattr(model, "control_key")
|
|
|
|
|
2023-01-28 01:18:42 +00:00
|
|
|
progress_latents = []
|
|
|
|
|
|
|
|
def latent_logger(latents):
|
|
|
|
progress_latents.append(latents)
|
|
|
|
|
2023-01-26 04:58:28 +00:00
|
|
|
with ImageLoggingContext(
|
|
|
|
prompt=prompt,
|
|
|
|
model=model,
|
|
|
|
debug_img_callback=debug_img_callback,
|
|
|
|
progress_img_callback=progress_img_callback,
|
|
|
|
progress_img_interval_steps=progress_img_interval_steps,
|
|
|
|
progress_img_interval_min_s=progress_img_interval_min_s,
|
2023-01-28 01:18:42 +00:00
|
|
|
progress_latent_callback=latent_logger
|
|
|
|
if prompt.collect_progress_latents
|
|
|
|
else None,
|
2023-01-26 04:58:28 +00:00
|
|
|
) as lc:
|
|
|
|
seed_everything(prompt.seed)
|
|
|
|
|
|
|
|
model.tile_mode(prompt.tile_mode)
|
|
|
|
with lc.timing("conditioning"):
|
|
|
|
# need to expand if doing batches
|
|
|
|
neutral_conditioning = _prompts_to_embeddings(prompt.negative_prompt, model)
|
2023-02-16 16:11:31 +00:00
|
|
|
_prompts_to_embeddings("", model)
|
2023-01-26 04:58:28 +00:00
|
|
|
log_conditioning(neutral_conditioning, "neutral conditioning")
|
|
|
|
if prompt.conditioning is not None:
|
|
|
|
positive_conditioning = prompt.conditioning
|
|
|
|
else:
|
|
|
|
positive_conditioning = _prompts_to_embeddings(prompt.prompts, model)
|
|
|
|
log_conditioning(positive_conditioning, "positive conditioning")
|
|
|
|
|
|
|
|
shape = [
|
|
|
|
batch_size,
|
|
|
|
latent_channels,
|
|
|
|
prompt.height // downsampling_factor,
|
|
|
|
prompt.width // downsampling_factor,
|
|
|
|
]
|
|
|
|
SamplerCls = SAMPLER_LOOKUP[prompt.sampler_type.lower()]
|
|
|
|
sampler = SamplerCls(model)
|
2023-02-15 16:02:36 +00:00
|
|
|
mask_latent = mask_image = mask_image_orig = mask_grayscale = None
|
2023-02-27 04:07:49 +00:00
|
|
|
t_enc = init_latent = control_image = None
|
2023-01-26 04:58:28 +00:00
|
|
|
starting_image = None
|
2023-02-15 16:02:36 +00:00
|
|
|
denoiser_cls = None
|
|
|
|
|
|
|
|
c_cat = []
|
|
|
|
c_cat_neutral = None
|
|
|
|
result_images = {}
|
2023-02-12 02:23:45 +00:00
|
|
|
seed_everything(prompt.seed)
|
|
|
|
noise = randn_seeded(seed=prompt.seed, size=shape).to(get_device())
|
|
|
|
|
2023-01-26 04:58:28 +00:00
|
|
|
if prompt.init_image:
|
|
|
|
starting_image = prompt.init_image
|
|
|
|
generation_strength = 1 - prompt.init_image_strength
|
2023-02-27 04:07:49 +00:00
|
|
|
|
|
|
|
if model.cond_stage_key == "edit" or generation_strength >= 1:
|
|
|
|
t_enc = None
|
2023-01-26 04:58:28 +00:00
|
|
|
else:
|
|
|
|
t_enc = int(prompt.steps * generation_strength)
|
|
|
|
|
|
|
|
if prompt.mask_prompt:
|
|
|
|
mask_image, mask_grayscale = get_img_mask(
|
|
|
|
starting_image, prompt.mask_prompt, threshold=0.1
|
|
|
|
)
|
|
|
|
elif prompt.mask_image:
|
|
|
|
mask_image = prompt.mask_image.convert("L")
|
|
|
|
if prompt.outpaint:
|
|
|
|
outpaint_kwargs = outpaint_arg_str_parse(prompt.outpaint)
|
|
|
|
starting_image, mask_image = prepare_image_for_outpaint(
|
|
|
|
starting_image, mask_image, **outpaint_kwargs
|
|
|
|
)
|
|
|
|
|
|
|
|
init_image = pillow_fit_image_within(
|
|
|
|
starting_image,
|
|
|
|
max_height=prompt.height,
|
|
|
|
max_width=prompt.width,
|
2022-10-24 05:42:17 +00:00
|
|
|
)
|
2023-01-17 06:48:27 +00:00
|
|
|
|
2023-01-26 04:58:28 +00:00
|
|
|
if mask_image is not None:
|
|
|
|
mask_image = pillow_fit_image_within(
|
|
|
|
mask_image,
|
|
|
|
max_height=prompt.height,
|
|
|
|
max_width=prompt.width,
|
|
|
|
convert="L",
|
|
|
|
)
|
|
|
|
|
|
|
|
log_img(mask_image, "init mask")
|
|
|
|
|
|
|
|
if prompt.mask_mode == ImaginePrompt.MaskMode.REPLACE:
|
|
|
|
mask_image = ImageOps.invert(mask_image)
|
|
|
|
|
|
|
|
mask_image_orig = mask_image
|
|
|
|
log_img(mask_image, "latent_mask")
|
2023-02-15 16:02:36 +00:00
|
|
|
mask_latent = pillow_mask_to_latent_mask(
|
|
|
|
mask_image, downsampling_factor=downsampling_factor
|
|
|
|
).to(get_device())
|
2023-01-26 04:58:28 +00:00
|
|
|
|
|
|
|
init_image_t = pillow_img_to_torch_image(init_image)
|
|
|
|
init_image_t = init_image_t.to(get_device())
|
|
|
|
init_latent = model.get_first_stage_encoding(
|
|
|
|
model.encode_first_stage(init_image_t)
|
|
|
|
)
|
|
|
|
shape = init_latent.shape
|
|
|
|
|
|
|
|
log_latent(init_latent, "init_latent")
|
|
|
|
seed_everything(prompt.seed)
|
2023-02-12 02:23:45 +00:00
|
|
|
noise = randn_seeded(seed=prompt.seed, size=init_latent.shape).to(
|
|
|
|
get_device()
|
|
|
|
)
|
|
|
|
# noise = noise[:, :, : init_latent.shape[2], : init_latent.shape[3]]
|
2023-01-26 04:58:28 +00:00
|
|
|
|
2023-02-27 04:07:49 +00:00
|
|
|
# schedule = NoiseSchedule(
|
|
|
|
# model_num_timesteps=model.num_timesteps,
|
|
|
|
# ddim_num_steps=prompt.steps,
|
|
|
|
# model_alphas_cumprod=model.alphas_cumprod,
|
|
|
|
# ddim_discretize="uniform",
|
|
|
|
# )
|
|
|
|
# if generation_strength >= 1:
|
|
|
|
# # prompt strength gets converted to time encodings,
|
|
|
|
# # which means you can't get to true 0 without this hack
|
|
|
|
# # (or setting steps=1000)
|
|
|
|
# init_latent_noised = noise
|
|
|
|
# else:
|
|
|
|
# init_latent_noised = noise_an_image(
|
|
|
|
# init_latent,
|
|
|
|
# torch.tensor([t_enc - 1]).to(get_device()),
|
|
|
|
# schedule=schedule,
|
|
|
|
# noise=noise,
|
|
|
|
# )
|
2023-02-15 16:02:36 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
if hasattr(model, "depth_stage_key"):
|
|
|
|
# depth model
|
|
|
|
depth_t = torch_image_to_depth_map(init_image_t)
|
|
|
|
depth_latent = torch.nn.functional.interpolate(
|
|
|
|
depth_t,
|
|
|
|
size=shape[2:],
|
|
|
|
mode="bicubic",
|
|
|
|
align_corners=False,
|
|
|
|
)
|
|
|
|
result_images["depth_image"] = depth_t
|
|
|
|
c_cat.append(depth_latent)
|
|
|
|
|
|
|
|
elif is_controlnet_model:
|
|
|
|
from imaginairy.img_processors.control_modes import CONTROL_MODES
|
|
|
|
|
|
|
|
if prompt.control_image_raw is not None:
|
|
|
|
control_image = prompt.control_image_raw
|
|
|
|
elif prompt.control_image is not None:
|
|
|
|
control_image = prompt.control_image
|
|
|
|
control_image = control_image.convert("RGB")
|
|
|
|
log_img(control_image, "control_image_input")
|
|
|
|
control_image_input = pillow_fit_image_within(
|
|
|
|
control_image,
|
|
|
|
max_height=prompt.height,
|
|
|
|
max_width=prompt.width,
|
|
|
|
)
|
|
|
|
control_image_input_t = pillow_img_to_torch_image(control_image_input)
|
|
|
|
control_image_input_t = control_image_input_t.to(get_device())
|
2023-02-12 07:42:19 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
if prompt.control_image_raw is None:
|
|
|
|
control_image_t = CONTROL_MODES[prompt.control_mode](
|
2023-02-12 07:42:19 +00:00
|
|
|
control_image_input_t
|
|
|
|
)
|
2023-02-25 09:57:43 +00:00
|
|
|
else:
|
|
|
|
control_image_t = (control_image_input_t + 1) / 2
|
2023-02-12 07:42:19 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
control_image_disp = control_image_t * 2 - 1
|
|
|
|
result_images["control"] = control_image_disp[:, [2, 0, 1], :, :]
|
|
|
|
log_img(control_image_disp, "control_image")
|
2023-02-12 07:42:19 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
if len(control_image_t.shape) == 3:
|
|
|
|
raise RuntimeError("Control image must be 4D")
|
2023-02-12 07:42:19 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
if control_image_t.shape[1] != 3:
|
|
|
|
raise RuntimeError("Control image must have 3 channels")
|
2023-02-12 07:42:19 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
if control_image_t.min() < 0 or control_image_t.max() > 1:
|
|
|
|
raise RuntimeError(
|
|
|
|
f"Control image must be in [0, 1] but we received {control_image_t.min()} and {control_image_t.max()}"
|
|
|
|
)
|
2023-02-12 07:42:19 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
if control_image_t.max() == control_image_t.min():
|
|
|
|
raise RuntimeError("No control signal found in control image.")
|
2023-02-12 07:42:19 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
c_cat.append(control_image_t)
|
2023-02-15 16:02:36 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
elif hasattr(model, "masked_image_key"):
|
|
|
|
# inpainting model
|
|
|
|
mask_t = pillow_img_to_torch_image(ImageOps.invert(mask_image_orig)).to(
|
|
|
|
get_device()
|
|
|
|
)
|
|
|
|
inverted_mask = 1 - mask_latent
|
|
|
|
masked_image_t = init_image_t * (mask_t < 0.5)
|
|
|
|
log_img(masked_image_t, "masked_image")
|
2022-10-23 21:46:45 +00:00
|
|
|
|
2023-02-25 09:57:43 +00:00
|
|
|
inverted_mask_latent = torch.nn.functional.interpolate(
|
|
|
|
inverted_mask, size=shape[-2:]
|
|
|
|
)
|
|
|
|
c_cat.append(inverted_mask_latent)
|
|
|
|
|
|
|
|
masked_image_latent = model.get_first_stage_encoding(
|
|
|
|
model.encode_first_stage(masked_image_t)
|
|
|
|
)
|
|
|
|
c_cat.append(masked_image_latent)
|
|
|
|
|
|
|
|
elif model.cond_stage_key == "edit":
|
|
|
|
# pix2pix model
|
|
|
|
c_cat = [model.encode_first_stage(init_image_t)]
|
|
|
|
c_cat_neutral = [torch.zeros_like(init_latent)]
|
|
|
|
denoiser_cls = CFGEditingDenoiser
|
|
|
|
if c_cat:
|
|
|
|
c_cat = [torch.cat(c_cat, dim=1)]
|
2023-01-26 04:58:28 +00:00
|
|
|
|
|
|
|
if c_cat_neutral is None:
|
|
|
|
c_cat_neutral = c_cat
|
|
|
|
|
|
|
|
positive_conditioning = {
|
|
|
|
"c_concat": c_cat,
|
|
|
|
"c_crossattn": [positive_conditioning],
|
|
|
|
}
|
|
|
|
neutral_conditioning = {
|
|
|
|
"c_concat": c_cat_neutral,
|
|
|
|
"c_crossattn": [neutral_conditioning],
|
|
|
|
}
|
2023-02-15 16:02:36 +00:00
|
|
|
|
2023-02-12 07:42:19 +00:00
|
|
|
if (
|
|
|
|
prompt.allow_compose_phase
|
|
|
|
and not is_controlnet_model
|
|
|
|
and not model.cond_stage_key == "edit"
|
|
|
|
):
|
2023-03-01 04:54:26 +00:00
|
|
|
if prompt.init_image:
|
|
|
|
comp_image = _generate_composition_image(
|
|
|
|
prompt=prompt,
|
|
|
|
target_height=init_image.height,
|
|
|
|
target_width=init_image.width,
|
|
|
|
)
|
|
|
|
else:
|
|
|
|
comp_image = _generate_composition_image(
|
|
|
|
prompt=prompt,
|
|
|
|
target_height=prompt.height,
|
|
|
|
target_width=prompt.width,
|
|
|
|
)
|
|
|
|
if comp_image is not None:
|
|
|
|
result_images["composition"] = comp_image
|
|
|
|
# noise = noise[:, :, : comp_image.height, : comp_image.shape[3]]
|
|
|
|
t_enc = int(prompt.steps * 0.65)
|
|
|
|
log_img(comp_image, "comp_image")
|
|
|
|
comp_image_t = pillow_img_to_torch_image(comp_image)
|
|
|
|
comp_image_t = comp_image_t.to(get_device())
|
|
|
|
init_latent = model.get_first_stage_encoding(
|
|
|
|
model.encode_first_stage(comp_image_t)
|
|
|
|
)
|
2023-01-26 04:58:28 +00:00
|
|
|
with lc.timing("sampling"):
|
|
|
|
samples = sampler.sample(
|
|
|
|
num_steps=prompt.steps,
|
|
|
|
positive_conditioning=positive_conditioning,
|
|
|
|
neutral_conditioning=neutral_conditioning,
|
|
|
|
guidance_scale=prompt.prompt_strength,
|
|
|
|
t_start=t_enc,
|
2023-02-15 16:02:36 +00:00
|
|
|
mask=mask_latent,
|
2023-01-26 04:58:28 +00:00
|
|
|
orig_latent=init_latent,
|
|
|
|
shape=shape,
|
|
|
|
batch_size=1,
|
|
|
|
denoiser_cls=denoiser_cls,
|
2023-02-27 04:07:49 +00:00
|
|
|
noise=noise,
|
2023-01-26 04:58:28 +00:00
|
|
|
)
|
2023-02-12 02:23:45 +00:00
|
|
|
if return_latent:
|
|
|
|
return samples
|
2023-02-15 16:02:36 +00:00
|
|
|
|
2023-01-26 11:14:02 +00:00
|
|
|
with lc.timing("decoding"):
|
2023-02-15 16:02:36 +00:00
|
|
|
gen_imgs_t = model.decode_first_stage(samples)
|
|
|
|
gen_img = torch_img_to_pillow_img(gen_imgs_t)
|
|
|
|
|
|
|
|
if mask_image_orig and init_image:
|
|
|
|
mask_final = mask_image_orig.copy()
|
|
|
|
log_img(mask_final, "reconstituting mask")
|
|
|
|
mask_final = ImageOps.invert(mask_final)
|
|
|
|
gen_img = Image.composite(gen_img, init_image, mask_final)
|
2023-02-12 02:23:45 +00:00
|
|
|
gen_img = combine_image(
|
|
|
|
original_img=init_image,
|
|
|
|
generated_img=gen_img,
|
|
|
|
mask_img=mask_image_orig,
|
|
|
|
)
|
2023-02-15 16:02:36 +00:00
|
|
|
log_img(gen_img, "reconstituted image")
|
|
|
|
|
|
|
|
upscaled_img = None
|
|
|
|
rebuilt_orig_img = None
|
|
|
|
|
|
|
|
if add_caption:
|
|
|
|
caption = generate_caption(gen_img)
|
|
|
|
logger.info(f"Generated caption: {caption}")
|
|
|
|
|
|
|
|
with lc.timing("safety-filter"):
|
|
|
|
safety_score = create_safety_score(
|
|
|
|
gen_img,
|
|
|
|
safety_mode=IMAGINAIRY_SAFETY_MODE,
|
2023-01-26 04:58:28 +00:00
|
|
|
)
|
2023-02-15 16:02:36 +00:00
|
|
|
if safety_score.is_filtered:
|
|
|
|
progress_latents.clear()
|
|
|
|
if not safety_score.is_filtered:
|
|
|
|
if prompt.fix_faces:
|
|
|
|
logger.info("Fixing 😊 's in 🖼 using CodeFormer...")
|
|
|
|
with lc.timing("face enhancement"):
|
|
|
|
gen_img = enhance_faces(gen_img, fidelity=prompt.fix_faces_fidelity)
|
|
|
|
if prompt.upscale:
|
|
|
|
logger.info("Upscaling 🖼 using real-ESRGAN...")
|
|
|
|
with lc.timing("upscaling"):
|
|
|
|
upscaled_img = upscale_image(gen_img)
|
|
|
|
|
|
|
|
# put the newly generated patch back into the original, full-size image
|
|
|
|
if prompt.mask_modify_original and mask_image_orig and starting_image:
|
|
|
|
img_to_add_back_to_original = upscaled_img if upscaled_img else gen_img
|
|
|
|
rebuilt_orig_img = combine_image(
|
|
|
|
original_img=starting_image,
|
|
|
|
generated_img=img_to_add_back_to_original,
|
|
|
|
mask_img=mask_image_orig,
|
|
|
|
)
|
|
|
|
|
2023-02-22 04:41:29 +00:00
|
|
|
if prompt.caption_text:
|
|
|
|
add_caption_to_image(gen_img, prompt.caption_text)
|
|
|
|
|
2023-02-15 16:02:36 +00:00
|
|
|
result = ImagineResult(
|
|
|
|
img=gen_img,
|
|
|
|
prompt=prompt,
|
|
|
|
upscaled_img=upscaled_img,
|
|
|
|
is_nsfw=safety_score.is_nsfw,
|
|
|
|
safety_score=safety_score,
|
|
|
|
modified_original=rebuilt_orig_img,
|
|
|
|
mask_binary=mask_image_orig,
|
|
|
|
mask_grayscale=mask_grayscale,
|
|
|
|
result_images=result_images,
|
|
|
|
timings=lc.get_timings(),
|
|
|
|
progress_latents=progress_latents.copy(),
|
|
|
|
)
|
|
|
|
|
|
|
|
_most_recent_result = result
|
|
|
|
logger.info(f"Image Generated. Timings: {result.timings_str()}")
|
|
|
|
return result
|
2022-09-08 03:59:30 +00:00
|
|
|
|
|
|
|
|
2022-12-02 09:49:13 +00:00
|
|
|
def _prompts_to_embeddings(prompts, model):
|
|
|
|
total_weight = sum(wp.weight for wp in prompts)
|
|
|
|
conditioning = sum(
|
|
|
|
model.get_learned_conditioning(wp.text) * (wp.weight / total_weight)
|
|
|
|
for wp in prompts
|
|
|
|
)
|
|
|
|
return conditioning
|
|
|
|
|
|
|
|
|
2023-02-12 02:23:45 +00:00
|
|
|
def calc_scale_to_fit_within(
|
|
|
|
height,
|
|
|
|
width,
|
|
|
|
max_size,
|
|
|
|
):
|
|
|
|
if max(height, width) < max_size:
|
|
|
|
return 1
|
|
|
|
|
|
|
|
if width > height:
|
|
|
|
return max_size / width
|
|
|
|
|
|
|
|
return max_size / height
|
|
|
|
|
|
|
|
|
2023-03-01 04:54:26 +00:00
|
|
|
def _scale_latent(
|
|
|
|
latent,
|
|
|
|
model,
|
|
|
|
h,
|
|
|
|
w,
|
|
|
|
):
|
|
|
|
from torch.nn import functional as F
|
|
|
|
|
|
|
|
# convert to non-latent-space first
|
|
|
|
img = model.decode_first_stage(latent)
|
|
|
|
img = F.interpolate(img, size=(h, w), mode="bicubic", align_corners=False)
|
|
|
|
latent = model.get_first_stage_encoding(model.encode_first_stage(img))
|
|
|
|
return latent
|
|
|
|
|
|
|
|
|
|
|
|
def _generate_composition_image(prompt, target_height, target_width):
|
|
|
|
from copy import copy
|
|
|
|
|
|
|
|
from PIL import Image
|
|
|
|
|
|
|
|
cutoff = 512
|
|
|
|
if prompt.width <= cutoff and prompt.height <= cutoff:
|
|
|
|
return None
|
|
|
|
|
|
|
|
composition_prompt = copy(prompt)
|
|
|
|
shrink_scale = calc_scale_to_fit_within(
|
|
|
|
height=prompt.height,
|
|
|
|
width=prompt.width,
|
|
|
|
max_size=cutoff,
|
|
|
|
)
|
|
|
|
composition_prompt.width = int(prompt.width * shrink_scale)
|
|
|
|
composition_prompt.height = int(prompt.height * shrink_scale)
|
|
|
|
|
|
|
|
composition_prompt.steps = None
|
|
|
|
composition_prompt.upscaled = False
|
|
|
|
composition_prompt.fix_faces = False
|
|
|
|
composition_prompt.mask_modify_original = False
|
|
|
|
|
|
|
|
composition_prompt.validate()
|
|
|
|
|
|
|
|
result = _generate_single_image(composition_prompt)
|
|
|
|
img = result.images["generated"]
|
|
|
|
while img.width < target_width:
|
|
|
|
from imaginairy.enhancers.upscale_realesrgan import upscale_image
|
|
|
|
|
|
|
|
img = upscale_image(img)
|
|
|
|
|
|
|
|
# samples = _generate_single_image(composition_prompt, return_latent=True)
|
|
|
|
# while samples.shape[-1] * 8 < target_width:
|
|
|
|
# samples = upscale_latent(samples)
|
|
|
|
#
|
|
|
|
# img = model_latent_to_pillow_img(samples)
|
|
|
|
|
|
|
|
img = img.resize(
|
|
|
|
(target_width, target_height),
|
|
|
|
resample=Image.Resampling.LANCZOS,
|
|
|
|
)
|
|
|
|
|
|
|
|
return img
|
|
|
|
|
|
|
|
|
|
|
|
def _generate_composition_latentz(
|
2023-02-12 02:23:45 +00:00
|
|
|
sampler,
|
|
|
|
sampler_kwargs,
|
|
|
|
):
|
|
|
|
from copy import deepcopy
|
|
|
|
|
|
|
|
from torch.nn import functional as F
|
|
|
|
|
2023-02-12 07:42:19 +00:00
|
|
|
from imaginairy.enhancers.upscale_riverwing import upscale_latent
|
2023-02-25 08:14:12 +00:00
|
|
|
from imaginairy.log_utils import log_img, log_latent
|
2023-02-12 07:42:19 +00:00
|
|
|
|
2023-02-27 04:07:49 +00:00
|
|
|
b, c, h, w = sampler_kwargs["shape"]
|
2023-02-12 02:23:45 +00:00
|
|
|
max_compose_gen_size = 768
|
|
|
|
shrink_scale = calc_scale_to_fit_within(
|
|
|
|
height=h,
|
|
|
|
width=w,
|
|
|
|
max_size=int(math.ceil(max_compose_gen_size / 8)),
|
|
|
|
)
|
|
|
|
if shrink_scale >= 1:
|
|
|
|
return None
|
|
|
|
|
2023-02-16 16:11:31 +00:00
|
|
|
new_kwargs = deepcopy(sampler_kwargs)
|
|
|
|
|
2023-02-12 02:23:45 +00:00
|
|
|
# shrink everything
|
|
|
|
new_shape = b, c, int(round(h * shrink_scale)), int(round(w * shrink_scale))
|
2023-02-27 04:07:49 +00:00
|
|
|
noise = new_kwargs["noise"]
|
|
|
|
if noise is not None:
|
|
|
|
noise = F.interpolate(noise, size=new_shape[2:], mode="nearest-exact")
|
2023-02-12 02:23:45 +00:00
|
|
|
|
|
|
|
for cond in [
|
|
|
|
new_kwargs["positive_conditioning"],
|
|
|
|
new_kwargs["neutral_conditioning"],
|
|
|
|
]:
|
2023-03-01 04:54:26 +00:00
|
|
|
print(cond["c_concat"])
|
|
|
|
for c in cond["c_concat"]:
|
|
|
|
print(f"downscaling {c.shape} ")
|
2023-02-12 02:23:45 +00:00
|
|
|
cond["c_concat"] = [
|
2023-03-01 04:54:26 +00:00
|
|
|
_scale_latent(
|
|
|
|
latent=c, model=sampler.model, h=new_shape[2] * 8, w=new_shape[3] * 8
|
|
|
|
)
|
|
|
|
for c in cond["c_concat"]
|
2023-02-12 02:23:45 +00:00
|
|
|
]
|
2023-03-01 04:54:26 +00:00
|
|
|
print(cond["c_concat"])
|
2023-02-12 02:23:45 +00:00
|
|
|
|
|
|
|
mask_latent = new_kwargs["mask"]
|
|
|
|
if mask_latent is not None:
|
|
|
|
mask_latent = F.interpolate(mask_latent, size=new_shape[2:], mode="area")
|
|
|
|
|
|
|
|
orig_latent = new_kwargs["orig_latent"]
|
|
|
|
if orig_latent is not None:
|
2023-03-01 04:54:26 +00:00
|
|
|
orig_latent = _scale_latent(
|
|
|
|
latent=orig_latent,
|
|
|
|
model=sampler.model,
|
|
|
|
h=new_shape[2] * 8,
|
|
|
|
w=new_shape[3] * 8,
|
|
|
|
)
|
|
|
|
|
2023-02-12 02:23:45 +00:00
|
|
|
t_start = new_kwargs["t_start"]
|
|
|
|
if t_start is not None:
|
|
|
|
gen_strength = new_kwargs["t_start"] / new_kwargs["num_steps"]
|
|
|
|
t_start = int(round(15 * gen_strength))
|
|
|
|
new_kwargs.update(
|
|
|
|
{
|
|
|
|
"num_steps": 15,
|
2023-02-27 04:07:49 +00:00
|
|
|
"noise": noise,
|
2023-02-12 02:23:45 +00:00
|
|
|
"t_start": t_start,
|
|
|
|
"mask": mask_latent,
|
|
|
|
"orig_latent": orig_latent,
|
|
|
|
"shape": new_shape,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
samples = sampler.sample(**new_kwargs)
|
2023-02-12 07:42:19 +00:00
|
|
|
|
2023-02-27 04:07:49 +00:00
|
|
|
# while samples.shape[2] < h:
|
|
|
|
logger.info("Upscaling latent...")
|
2023-02-16 16:11:31 +00:00
|
|
|
samples = upscale_latent(samples)
|
2023-02-25 08:14:12 +00:00
|
|
|
log_latent(samples, "upscaled")
|
|
|
|
img_t = sampler.model.decode_first_stage(samples)
|
|
|
|
|
|
|
|
img_t = F.interpolate(img_t, size=(h * 8, w * 8), mode="bicubic")
|
|
|
|
log_img(img_t, "upscaled interpolated")
|
|
|
|
samples = sampler.model.get_first_stage_encoding(
|
|
|
|
sampler.model.encode_first_stage(img_t)
|
|
|
|
)
|
|
|
|
log_latent(samples, "upscaled interpolated latent")
|
|
|
|
|
2023-02-12 02:23:45 +00:00
|
|
|
return samples
|
|
|
|
|
|
|
|
|
2022-09-08 03:59:30 +00:00
|
|
|
def prompt_normalized(prompt):
|
2022-09-25 20:07:27 +00:00
|
|
|
return re.sub(r"[^a-zA-Z0-9.,\[\]-]+", "_", prompt)[:130]
|
2023-02-15 16:02:36 +00:00
|
|
|
|
|
|
|
|
|
|
|
def combine_image(original_img, generated_img, mask_img):
|
|
|
|
"""Combine the generated image with the original image using the mask image."""
|
|
|
|
from PIL import Image
|
|
|
|
|
|
|
|
from imaginairy.log_utils import log_img
|
|
|
|
|
|
|
|
generated_img = generated_img.resize(
|
|
|
|
original_img.size,
|
|
|
|
resample=Image.Resampling.LANCZOS,
|
|
|
|
)
|
|
|
|
|
|
|
|
mask_for_orig_size = mask_img.resize(
|
|
|
|
original_img.size,
|
|
|
|
resample=Image.Resampling.LANCZOS,
|
|
|
|
)
|
|
|
|
log_img(mask_for_orig_size, "mask for original image size")
|
|
|
|
|
|
|
|
rebuilt_orig_img = Image.composite(
|
|
|
|
original_img,
|
|
|
|
generated_img,
|
|
|
|
mask_for_orig_size,
|
|
|
|
)
|
|
|
|
log_img(rebuilt_orig_img, "reconstituted original")
|
|
|
|
return rebuilt_orig_img
|