You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
375 lines
17 KiB
Python
375 lines
17 KiB
Python
import cv2
|
|
import numpy as np
|
|
import os
|
|
import torch
|
|
from torchvision.transforms.functional import normalize
|
|
|
|
from imaginairy.vendored.facexlib.detection import init_detection_model
|
|
from imaginairy.vendored.facexlib.parsing import init_parsing_model
|
|
from imaginairy.vendored.facexlib.utils.misc import img2tensor, imwrite
|
|
|
|
|
|
def get_largest_face(det_faces, h, w):
|
|
|
|
def get_location(val, length):
|
|
if val < 0:
|
|
return 0
|
|
elif val > length:
|
|
return length
|
|
else:
|
|
return val
|
|
|
|
face_areas = []
|
|
for det_face in det_faces:
|
|
left = get_location(det_face[0], w)
|
|
right = get_location(det_face[2], w)
|
|
top = get_location(det_face[1], h)
|
|
bottom = get_location(det_face[3], h)
|
|
face_area = (right - left) * (bottom - top)
|
|
face_areas.append(face_area)
|
|
largest_idx = face_areas.index(max(face_areas))
|
|
return det_faces[largest_idx], largest_idx
|
|
|
|
|
|
def get_center_face(det_faces, h=0, w=0, center=None):
|
|
if center is not None:
|
|
center = np.array(center)
|
|
else:
|
|
center = np.array([w / 2, h / 2])
|
|
center_dist = []
|
|
for det_face in det_faces:
|
|
face_center = np.array([(det_face[0] + det_face[2]) / 2, (det_face[1] + det_face[3]) / 2])
|
|
dist = np.linalg.norm(face_center - center)
|
|
center_dist.append(dist)
|
|
center_idx = center_dist.index(min(center_dist))
|
|
return det_faces[center_idx], center_idx
|
|
|
|
|
|
class FaceRestoreHelper(object):
|
|
"""Helper for the face restoration pipeline (base class)."""
|
|
|
|
def __init__(self,
|
|
upscale_factor,
|
|
face_size=512,
|
|
crop_ratio=(1, 1),
|
|
det_model='retinaface_resnet50',
|
|
save_ext='png',
|
|
template_3points=False,
|
|
pad_blur=False,
|
|
use_parse=False,
|
|
device=None,
|
|
model_rootpath=None):
|
|
self.template_3points = template_3points # improve robustness
|
|
self.upscale_factor = upscale_factor
|
|
# the cropped face ratio based on the square face
|
|
self.crop_ratio = crop_ratio # (h, w)
|
|
assert (self.crop_ratio[0] >= 1 and self.crop_ratio[1] >= 1), 'crop ration only supports >=1'
|
|
self.face_size = (int(face_size * self.crop_ratio[1]), int(face_size * self.crop_ratio[0]))
|
|
|
|
if self.template_3points:
|
|
self.face_template = np.array([[192, 240], [319, 240], [257, 371]])
|
|
else:
|
|
# standard 5 landmarks for FFHQ faces with 512 x 512
|
|
self.face_template = np.array([[192.98138, 239.94708], [318.90277, 240.1936], [256.63416, 314.01935],
|
|
[201.26117, 371.41043], [313.08905, 371.15118]])
|
|
self.face_template = self.face_template * (face_size / 512.0)
|
|
if self.crop_ratio[0] > 1:
|
|
self.face_template[:, 1] += face_size * (self.crop_ratio[0] - 1) / 2
|
|
if self.crop_ratio[1] > 1:
|
|
self.face_template[:, 0] += face_size * (self.crop_ratio[1] - 1) / 2
|
|
self.save_ext = save_ext
|
|
self.pad_blur = pad_blur
|
|
if self.pad_blur is True:
|
|
self.template_3points = False
|
|
|
|
self.all_landmarks_5 = []
|
|
self.det_faces = []
|
|
self.affine_matrices = []
|
|
self.inverse_affine_matrices = []
|
|
self.cropped_faces = []
|
|
self.restored_faces = []
|
|
self.pad_input_imgs = []
|
|
|
|
if device is None:
|
|
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
|
else:
|
|
self.device = device
|
|
|
|
# init face detection model
|
|
self.face_det = init_detection_model(det_model, half=False, device=self.device, model_rootpath=model_rootpath)
|
|
|
|
# init face parsing model
|
|
self.use_parse = use_parse
|
|
self.face_parse = init_parsing_model(model_name='parsenet', device=self.device, model_rootpath=model_rootpath)
|
|
|
|
def set_upscale_factor(self, upscale_factor):
|
|
self.upscale_factor = upscale_factor
|
|
|
|
def read_image(self, img):
|
|
"""img can be image path or cv2 loaded image."""
|
|
# self.input_img is Numpy array, (h, w, c), BGR, uint8, [0, 255]
|
|
if isinstance(img, str):
|
|
img = cv2.imread(img)
|
|
|
|
if np.max(img) > 256: # 16-bit image
|
|
img = img / 65535 * 255
|
|
if len(img.shape) == 2: # gray image
|
|
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
|
|
elif img.shape[2] == 4: # RGBA image with alpha channel
|
|
img = img[:, :, 0:3]
|
|
|
|
self.input_img = img
|
|
|
|
def get_face_landmarks_5(self,
|
|
only_keep_largest=False,
|
|
only_center_face=False,
|
|
resize=None,
|
|
blur_ratio=0.01,
|
|
eye_dist_threshold=None):
|
|
if resize is None:
|
|
scale = 1
|
|
input_img = self.input_img
|
|
else:
|
|
h, w = self.input_img.shape[0:2]
|
|
scale = min(h, w) / resize
|
|
h, w = int(h / scale), int(w / scale)
|
|
input_img = cv2.resize(self.input_img, (w, h), interpolation=cv2.INTER_LANCZOS4)
|
|
|
|
with torch.no_grad():
|
|
bboxes = self.face_det.detect_faces(input_img, 0.97) * scale
|
|
for bbox in bboxes:
|
|
# remove faces with too small eye distance: side faces or too small faces
|
|
eye_dist = np.linalg.norm([bbox[5] - bbox[7], bbox[6] - bbox[8]])
|
|
if eye_dist_threshold is not None and (eye_dist < eye_dist_threshold):
|
|
continue
|
|
|
|
if self.template_3points:
|
|
landmark = np.array([[bbox[i], bbox[i + 1]] for i in range(5, 11, 2)])
|
|
else:
|
|
landmark = np.array([[bbox[i], bbox[i + 1]] for i in range(5, 15, 2)])
|
|
self.all_landmarks_5.append(landmark)
|
|
self.det_faces.append(bbox[0:5])
|
|
if len(self.det_faces) == 0:
|
|
return 0
|
|
if only_keep_largest:
|
|
h, w, _ = self.input_img.shape
|
|
self.det_faces, largest_idx = get_largest_face(self.det_faces, h, w)
|
|
self.all_landmarks_5 = [self.all_landmarks_5[largest_idx]]
|
|
elif only_center_face:
|
|
h, w, _ = self.input_img.shape
|
|
self.det_faces, center_idx = get_center_face(self.det_faces, h, w)
|
|
self.all_landmarks_5 = [self.all_landmarks_5[center_idx]]
|
|
|
|
# pad blurry images
|
|
if self.pad_blur:
|
|
self.pad_input_imgs = []
|
|
for landmarks in self.all_landmarks_5:
|
|
# get landmarks
|
|
eye_left = landmarks[0, :]
|
|
eye_right = landmarks[1, :]
|
|
eye_avg = (eye_left + eye_right) * 0.5
|
|
mouth_avg = (landmarks[3, :] + landmarks[4, :]) * 0.5
|
|
eye_to_eye = eye_right - eye_left
|
|
eye_to_mouth = mouth_avg - eye_avg
|
|
|
|
# Get the oriented crop rectangle
|
|
# x: half width of the oriented crop rectangle
|
|
x = eye_to_eye - np.flipud(eye_to_mouth) * [-1, 1]
|
|
# - np.flipud(eye_to_mouth) * [-1, 1]: rotate 90 clockwise
|
|
# norm with the hypotenuse: get the direction
|
|
x /= np.hypot(*x) # get the hypotenuse of a right triangle
|
|
rect_scale = 1.5
|
|
x *= max(np.hypot(*eye_to_eye) * 2.0 * rect_scale, np.hypot(*eye_to_mouth) * 1.8 * rect_scale)
|
|
# y: half height of the oriented crop rectangle
|
|
y = np.flipud(x) * [-1, 1]
|
|
|
|
# c: center
|
|
c = eye_avg + eye_to_mouth * 0.1
|
|
# quad: (left_top, left_bottom, right_bottom, right_top)
|
|
quad = np.stack([c - x - y, c - x + y, c + x + y, c + x - y])
|
|
# qsize: side length of the square
|
|
qsize = np.hypot(*x) * 2
|
|
border = max(int(np.rint(qsize * 0.1)), 3)
|
|
|
|
# get pad
|
|
# pad: (width_left, height_top, width_right, height_bottom)
|
|
pad = (int(np.floor(min(quad[:, 0]))), int(np.floor(min(quad[:, 1]))), int(np.ceil(max(quad[:, 0]))),
|
|
int(np.ceil(max(quad[:, 1]))))
|
|
pad = [
|
|
max(-pad[0] + border, 1),
|
|
max(-pad[1] + border, 1),
|
|
max(pad[2] - self.input_img.shape[0] + border, 1),
|
|
max(pad[3] - self.input_img.shape[1] + border, 1)
|
|
]
|
|
|
|
if max(pad) > 1:
|
|
# pad image
|
|
pad_img = np.pad(self.input_img, ((pad[1], pad[3]), (pad[0], pad[2]), (0, 0)), 'reflect')
|
|
# modify landmark coords
|
|
landmarks[:, 0] += pad[0]
|
|
landmarks[:, 1] += pad[1]
|
|
# blur pad images
|
|
h, w, _ = pad_img.shape
|
|
y, x, _ = np.ogrid[:h, :w, :1]
|
|
mask = np.maximum(1.0 - np.minimum(np.float32(x) / pad[0],
|
|
np.float32(w - 1 - x) / pad[2]),
|
|
1.0 - np.minimum(np.float32(y) / pad[1],
|
|
np.float32(h - 1 - y) / pad[3]))
|
|
blur = int(qsize * blur_ratio)
|
|
if blur % 2 == 0:
|
|
blur += 1
|
|
blur_img = cv2.boxFilter(pad_img, 0, ksize=(blur, blur))
|
|
# blur_img = cv2.GaussianBlur(pad_img, (blur, blur), 0)
|
|
|
|
pad_img = pad_img.astype('float32')
|
|
pad_img += (blur_img - pad_img) * np.clip(mask * 3.0 + 1.0, 0.0, 1.0)
|
|
pad_img += (np.median(pad_img, axis=(0, 1)) - pad_img) * np.clip(mask, 0.0, 1.0)
|
|
pad_img = np.clip(pad_img, 0, 255) # float32, [0, 255]
|
|
self.pad_input_imgs.append(pad_img)
|
|
else:
|
|
self.pad_input_imgs.append(np.copy(self.input_img))
|
|
|
|
return len(self.all_landmarks_5)
|
|
|
|
def align_warp_face(self, save_cropped_path=None, border_mode='constant'):
|
|
"""Align and warp faces with face template.
|
|
"""
|
|
if self.pad_blur:
|
|
assert len(self.pad_input_imgs) == len(
|
|
self.all_landmarks_5), f'Mismatched samples: {len(self.pad_input_imgs)} and {len(self.all_landmarks_5)}'
|
|
for idx, landmark in enumerate(self.all_landmarks_5):
|
|
# use 5 landmarks to get affine matrix
|
|
# use cv2.LMEDS method for the equivalence to skimage transform
|
|
# ref: https://blog.csdn.net/yichxi/article/details/115827338
|
|
affine_matrix = cv2.estimateAffinePartial2D(landmark, self.face_template, method=cv2.LMEDS)[0]
|
|
self.affine_matrices.append(affine_matrix)
|
|
# warp and crop faces
|
|
if border_mode == 'constant':
|
|
border_mode = cv2.BORDER_CONSTANT
|
|
elif border_mode == 'reflect101':
|
|
border_mode = cv2.BORDER_REFLECT101
|
|
elif border_mode == 'reflect':
|
|
border_mode = cv2.BORDER_REFLECT
|
|
if self.pad_blur:
|
|
input_img = self.pad_input_imgs[idx]
|
|
else:
|
|
input_img = self.input_img
|
|
cropped_face = cv2.warpAffine(
|
|
input_img, affine_matrix, self.face_size, borderMode=border_mode, borderValue=(135, 133, 132)) # gray
|
|
self.cropped_faces.append(cropped_face)
|
|
# save the cropped face
|
|
if save_cropped_path is not None:
|
|
path = os.path.splitext(save_cropped_path)[0]
|
|
save_path = f'{path}_{idx:02d}.{self.save_ext}'
|
|
imwrite(cropped_face, save_path)
|
|
|
|
def get_inverse_affine(self, save_inverse_affine_path=None):
|
|
"""Get inverse affine matrix."""
|
|
for idx, affine_matrix in enumerate(self.affine_matrices):
|
|
inverse_affine = cv2.invertAffineTransform(affine_matrix)
|
|
inverse_affine *= self.upscale_factor
|
|
self.inverse_affine_matrices.append(inverse_affine)
|
|
# save inverse affine matrices
|
|
if save_inverse_affine_path is not None:
|
|
path, _ = os.path.splitext(save_inverse_affine_path)
|
|
save_path = f'{path}_{idx:02d}.pth'
|
|
torch.save(inverse_affine, save_path)
|
|
|
|
def add_restored_face(self, face):
|
|
self.restored_faces.append(face)
|
|
|
|
def paste_faces_to_input_image(self, save_path=None, upsample_img=None):
|
|
h, w, _ = self.input_img.shape
|
|
h_up, w_up = int(h * self.upscale_factor), int(w * self.upscale_factor)
|
|
|
|
if upsample_img is None:
|
|
# simply resize the background
|
|
upsample_img = cv2.resize(self.input_img, (w_up, h_up), interpolation=cv2.INTER_LANCZOS4)
|
|
else:
|
|
upsample_img = cv2.resize(upsample_img, (w_up, h_up), interpolation=cv2.INTER_LANCZOS4)
|
|
|
|
assert len(self.restored_faces) == len(
|
|
self.inverse_affine_matrices), ('length of restored_faces and affine_matrices are different.')
|
|
for restored_face, inverse_affine in zip(self.restored_faces, self.inverse_affine_matrices):
|
|
# Add an offset to inverse affine matrix, for more precise back alignment
|
|
if self.upscale_factor > 1:
|
|
extra_offset = 0.5 * self.upscale_factor
|
|
else:
|
|
extra_offset = 0
|
|
inverse_affine[:, 2] += extra_offset
|
|
inv_restored = cv2.warpAffine(restored_face, inverse_affine, (w_up, h_up))
|
|
|
|
if self.use_parse:
|
|
# inference
|
|
face_input = cv2.resize(restored_face, (512, 512), interpolation=cv2.INTER_LINEAR)
|
|
face_input = img2tensor(face_input.astype('float32') / 255., bgr2rgb=True, float32=True)
|
|
normalize(face_input, (0.5, 0.5, 0.5), (0.5, 0.5, 0.5), inplace=True)
|
|
face_input = torch.unsqueeze(face_input, 0).to(self.device)
|
|
with torch.no_grad():
|
|
out = self.face_parse(face_input)[0]
|
|
out = out.argmax(dim=1).squeeze().cpu().numpy()
|
|
|
|
mask = np.zeros(out.shape)
|
|
MASK_COLORMAP = [0, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0, 255, 0, 0, 0]
|
|
for idx, color in enumerate(MASK_COLORMAP):
|
|
mask[out == idx] = color
|
|
# blur the mask
|
|
mask = cv2.GaussianBlur(mask, (101, 101), 11)
|
|
mask = cv2.GaussianBlur(mask, (101, 101), 11)
|
|
# remove the black borders
|
|
thres = 10
|
|
mask[:thres, :] = 0
|
|
mask[-thres:, :] = 0
|
|
mask[:, :thres] = 0
|
|
mask[:, -thres:] = 0
|
|
mask = mask / 255.
|
|
|
|
mask = cv2.resize(mask, restored_face.shape[:2])
|
|
mask = cv2.warpAffine(mask, inverse_affine, (w_up, h_up), flags=3)
|
|
inv_soft_mask = mask[:, :, None]
|
|
pasted_face = inv_restored
|
|
|
|
else: # use square parse maps
|
|
mask = np.ones(self.face_size, dtype=np.float32)
|
|
inv_mask = cv2.warpAffine(mask, inverse_affine, (w_up, h_up))
|
|
# remove the black borders
|
|
inv_mask_erosion = cv2.erode(
|
|
inv_mask, np.ones((int(2 * self.upscale_factor), int(2 * self.upscale_factor)), np.uint8))
|
|
pasted_face = inv_mask_erosion[:, :, None] * inv_restored
|
|
total_face_area = np.sum(inv_mask_erosion) # // 3
|
|
# compute the fusion edge based on the area of face
|
|
w_edge = int(total_face_area**0.5) // 20
|
|
erosion_radius = w_edge * 2
|
|
inv_mask_center = cv2.erode(inv_mask_erosion, np.ones((erosion_radius, erosion_radius), np.uint8))
|
|
blur_size = w_edge * 2
|
|
inv_soft_mask = cv2.GaussianBlur(inv_mask_center, (blur_size + 1, blur_size + 1), 0)
|
|
if len(upsample_img.shape) == 2: # upsample_img is gray image
|
|
upsample_img = upsample_img[:, :, None]
|
|
inv_soft_mask = inv_soft_mask[:, :, None]
|
|
|
|
if len(upsample_img.shape) == 3 and upsample_img.shape[2] == 4: # alpha channel
|
|
alpha = upsample_img[:, :, 3:]
|
|
upsample_img = inv_soft_mask * pasted_face + (1 - inv_soft_mask) * upsample_img[:, :, 0:3]
|
|
upsample_img = np.concatenate((upsample_img, alpha), axis=2)
|
|
else:
|
|
upsample_img = inv_soft_mask * pasted_face + (1 - inv_soft_mask) * upsample_img
|
|
|
|
if np.max(upsample_img) > 256: # 16-bit image
|
|
upsample_img = upsample_img.astype(np.uint16)
|
|
else:
|
|
upsample_img = upsample_img.astype(np.uint8)
|
|
if save_path is not None:
|
|
path = os.path.splitext(save_path)[0]
|
|
save_path = f'{path}.{self.save_ext}'
|
|
imwrite(upsample_img, save_path)
|
|
return upsample_img
|
|
|
|
def clean_all(self):
|
|
self.all_landmarks_5 = []
|
|
self.restored_faces = []
|
|
self.affine_matrices = []
|
|
self.cropped_faces = []
|
|
self.inverse_affine_matrices = []
|
|
self.det_faces = []
|
|
self.pad_input_imgs = []
|