You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
299 lines
11 KiB
Python
299 lines
11 KiB
Python
import torch as torch
|
|
import torch.nn as nn
|
|
from torch.nn import functional as F
|
|
|
|
|
|
class HyperIQA(nn.Module):
|
|
"""
|
|
Combine the hypernet and target network within a network.
|
|
"""
|
|
|
|
def __init__(self, *args):
|
|
super(HyperIQA, self).__init__()
|
|
self.hypernet = HyperNet(*args)
|
|
|
|
def forward(self, img):
|
|
net_params = self.hypernet(img)
|
|
# build the target network
|
|
target_net = TargetNet(net_params)
|
|
for param in target_net.parameters():
|
|
param.requires_grad = False
|
|
# predict the face quality
|
|
pred = target_net(net_params['target_in_vec'])
|
|
return pred
|
|
|
|
|
|
class HyperNet(nn.Module):
|
|
"""
|
|
Hyper network for learning perceptual rules.
|
|
Args:
|
|
lda_out_channels: local distortion aware module output size.
|
|
hyper_in_channels: input feature channels for hyper network.
|
|
target_in_size: input vector size for target network.
|
|
target_fc(i)_size: fully connection layer size of target network.
|
|
feature_size: input feature map width/height for hyper network.
|
|
Note:
|
|
For size match, input args must satisfy: 'target_fc(i)_size * target_fc(i+1)_size' is divisible by 'feature_size ^ 2'. # noqa E501
|
|
"""
|
|
|
|
def __init__(self, lda_out_channels, hyper_in_channels, target_in_size, target_fc1_size, target_fc2_size,
|
|
target_fc3_size, target_fc4_size, feature_size):
|
|
super(HyperNet, self).__init__()
|
|
|
|
self.hyperInChn = hyper_in_channels
|
|
self.target_in_size = target_in_size
|
|
self.f1 = target_fc1_size
|
|
self.f2 = target_fc2_size
|
|
self.f3 = target_fc3_size
|
|
self.f4 = target_fc4_size
|
|
self.feature_size = feature_size
|
|
|
|
self.res = resnet50_backbone(lda_out_channels, target_in_size)
|
|
|
|
self.pool = nn.AdaptiveAvgPool2d((1, 1))
|
|
|
|
# Conv layers for resnet output features
|
|
self.conv1 = nn.Sequential(
|
|
nn.Conv2d(2048, 1024, 1, padding=(0, 0)), nn.ReLU(inplace=True), nn.Conv2d(1024, 512, 1, padding=(0, 0)),
|
|
nn.ReLU(inplace=True), nn.Conv2d(512, self.hyperInChn, 1, padding=(0, 0)), nn.ReLU(inplace=True))
|
|
|
|
# Hyper network part, conv for generating target fc weights, fc for generating target fc biases
|
|
self.fc1w_conv = nn.Conv2d(
|
|
self.hyperInChn, int(self.target_in_size * self.f1 / feature_size**2), 3, padding=(1, 1))
|
|
self.fc1b_fc = nn.Linear(self.hyperInChn, self.f1)
|
|
|
|
self.fc2w_conv = nn.Conv2d(self.hyperInChn, int(self.f1 * self.f2 / feature_size**2), 3, padding=(1, 1))
|
|
self.fc2b_fc = nn.Linear(self.hyperInChn, self.f2)
|
|
|
|
self.fc3w_conv = nn.Conv2d(self.hyperInChn, int(self.f2 * self.f3 / feature_size**2), 3, padding=(1, 1))
|
|
self.fc3b_fc = nn.Linear(self.hyperInChn, self.f3)
|
|
|
|
self.fc4w_conv = nn.Conv2d(self.hyperInChn, int(self.f3 * self.f4 / feature_size**2), 3, padding=(1, 1))
|
|
self.fc4b_fc = nn.Linear(self.hyperInChn, self.f4)
|
|
|
|
self.fc5w_fc = nn.Linear(self.hyperInChn, self.f4)
|
|
self.fc5b_fc = nn.Linear(self.hyperInChn, 1)
|
|
|
|
def forward(self, img):
|
|
feature_size = self.feature_size
|
|
|
|
res_out = self.res(img)
|
|
|
|
# input vector for target net
|
|
target_in_vec = res_out['target_in_vec'].view(-1, self.target_in_size, 1, 1)
|
|
|
|
# input features for hyper net
|
|
hyper_in_feat = self.conv1(res_out['hyper_in_feat']).view(-1, self.hyperInChn, feature_size, feature_size)
|
|
|
|
# generating target net weights & biases
|
|
target_fc1w = self.fc1w_conv(hyper_in_feat).view(-1, self.f1, self.target_in_size, 1, 1)
|
|
target_fc1b = self.fc1b_fc(self.pool(hyper_in_feat).squeeze()).view(-1, self.f1)
|
|
|
|
target_fc2w = self.fc2w_conv(hyper_in_feat).view(-1, self.f2, self.f1, 1, 1)
|
|
target_fc2b = self.fc2b_fc(self.pool(hyper_in_feat).squeeze()).view(-1, self.f2)
|
|
|
|
target_fc3w = self.fc3w_conv(hyper_in_feat).view(-1, self.f3, self.f2, 1, 1)
|
|
target_fc3b = self.fc3b_fc(self.pool(hyper_in_feat).squeeze()).view(-1, self.f3)
|
|
|
|
target_fc4w = self.fc4w_conv(hyper_in_feat).view(-1, self.f4, self.f3, 1, 1)
|
|
target_fc4b = self.fc4b_fc(self.pool(hyper_in_feat).squeeze()).view(-1, self.f4)
|
|
|
|
target_fc5w = self.fc5w_fc(self.pool(hyper_in_feat).squeeze()).view(-1, 1, self.f4, 1, 1)
|
|
target_fc5b = self.fc5b_fc(self.pool(hyper_in_feat).squeeze()).view(-1, 1)
|
|
|
|
out = {}
|
|
out['target_in_vec'] = target_in_vec
|
|
out['target_fc1w'] = target_fc1w
|
|
out['target_fc1b'] = target_fc1b
|
|
out['target_fc2w'] = target_fc2w
|
|
out['target_fc2b'] = target_fc2b
|
|
out['target_fc3w'] = target_fc3w
|
|
out['target_fc3b'] = target_fc3b
|
|
out['target_fc4w'] = target_fc4w
|
|
out['target_fc4b'] = target_fc4b
|
|
out['target_fc5w'] = target_fc5w
|
|
out['target_fc5b'] = target_fc5b
|
|
|
|
return out
|
|
|
|
|
|
class Bottleneck(nn.Module):
|
|
expansion = 4
|
|
|
|
def __init__(self, inplanes, planes, stride=1, downsample=None):
|
|
super(Bottleneck, self).__init__()
|
|
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
|
|
self.bn1 = nn.BatchNorm2d(planes)
|
|
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
|
|
self.bn2 = nn.BatchNorm2d(planes)
|
|
self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
|
|
self.bn3 = nn.BatchNorm2d(planes * 4)
|
|
self.relu = nn.ReLU(inplace=True)
|
|
self.downsample = downsample
|
|
self.stride = stride
|
|
|
|
def forward(self, x):
|
|
residual = x
|
|
|
|
out = self.conv1(x)
|
|
out = self.bn1(out)
|
|
out = self.relu(out)
|
|
|
|
out = self.conv2(out)
|
|
out = self.bn2(out)
|
|
out = self.relu(out)
|
|
|
|
out = self.conv3(out)
|
|
out = self.bn3(out)
|
|
|
|
if self.downsample is not None:
|
|
residual = self.downsample(x)
|
|
|
|
out += residual
|
|
out = self.relu(out)
|
|
|
|
return out
|
|
|
|
|
|
class ResNetBackbone(nn.Module):
|
|
|
|
def __init__(self, lda_out_channels, in_chn, block, layers, num_classes=1000):
|
|
super(ResNetBackbone, self).__init__()
|
|
self.inplanes = 64
|
|
self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)
|
|
self.bn1 = nn.BatchNorm2d(64)
|
|
self.relu = nn.ReLU(inplace=True)
|
|
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
|
|
self.layer1 = self._make_layer(block, 64, layers[0])
|
|
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
|
|
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
|
|
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
|
|
|
|
# local distortion aware module
|
|
self.lda1_pool = nn.Sequential(
|
|
nn.Conv2d(256, 16, kernel_size=1, stride=1, padding=0, bias=False),
|
|
nn.AvgPool2d(7, stride=7),
|
|
)
|
|
self.lda1_fc = nn.Linear(16 * 64, lda_out_channels)
|
|
|
|
self.lda2_pool = nn.Sequential(
|
|
nn.Conv2d(512, 32, kernel_size=1, stride=1, padding=0, bias=False),
|
|
nn.AvgPool2d(7, stride=7),
|
|
)
|
|
self.lda2_fc = nn.Linear(32 * 16, lda_out_channels)
|
|
|
|
self.lda3_pool = nn.Sequential(
|
|
nn.Conv2d(1024, 64, kernel_size=1, stride=1, padding=0, bias=False),
|
|
nn.AvgPool2d(7, stride=7),
|
|
)
|
|
self.lda3_fc = nn.Linear(64 * 4, lda_out_channels)
|
|
|
|
self.lda4_pool = nn.AvgPool2d(7, stride=7)
|
|
self.lda4_fc = nn.Linear(2048, in_chn - lda_out_channels * 3)
|
|
|
|
def _make_layer(self, block, planes, blocks, stride=1):
|
|
downsample = None
|
|
if stride != 1 or self.inplanes != planes * block.expansion:
|
|
downsample = nn.Sequential(
|
|
nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False),
|
|
nn.BatchNorm2d(planes * block.expansion),
|
|
)
|
|
|
|
layers = []
|
|
layers.append(block(self.inplanes, planes, stride, downsample))
|
|
self.inplanes = planes * block.expansion
|
|
for i in range(1, blocks):
|
|
layers.append(block(self.inplanes, planes))
|
|
|
|
return nn.Sequential(*layers)
|
|
|
|
def forward(self, x):
|
|
x = self.conv1(x)
|
|
x = self.bn1(x)
|
|
x = self.relu(x)
|
|
x = self.maxpool(x)
|
|
x = self.layer1(x)
|
|
|
|
# the same effect as lda operation in the paper, but save much more memory
|
|
lda_1 = self.lda1_fc(self.lda1_pool(x).view(x.size(0), -1))
|
|
x = self.layer2(x)
|
|
lda_2 = self.lda2_fc(self.lda2_pool(x).view(x.size(0), -1))
|
|
x = self.layer3(x)
|
|
lda_3 = self.lda3_fc(self.lda3_pool(x).view(x.size(0), -1))
|
|
x = self.layer4(x)
|
|
lda_4 = self.lda4_fc(self.lda4_pool(x).view(x.size(0), -1))
|
|
|
|
vec = torch.cat((lda_1, lda_2, lda_3, lda_4), 1)
|
|
|
|
out = {}
|
|
out['hyper_in_feat'] = x
|
|
out['target_in_vec'] = vec
|
|
|
|
return out
|
|
|
|
|
|
def resnet50_backbone(lda_out_channels, in_chn, **kwargs):
|
|
"""Constructs a ResNet-50 model_hyper."""
|
|
model = ResNetBackbone(lda_out_channels, in_chn, Bottleneck, [3, 4, 6, 3], **kwargs)
|
|
return model
|
|
|
|
|
|
class TargetNet(nn.Module):
|
|
"""
|
|
Target network for quality prediction.
|
|
"""
|
|
|
|
def __init__(self, paras):
|
|
super(TargetNet, self).__init__()
|
|
self.l1 = nn.Sequential(
|
|
TargetFC(paras['target_fc1w'], paras['target_fc1b']),
|
|
nn.Sigmoid(),
|
|
)
|
|
self.l2 = nn.Sequential(
|
|
TargetFC(paras['target_fc2w'], paras['target_fc2b']),
|
|
nn.Sigmoid(),
|
|
)
|
|
|
|
self.l3 = nn.Sequential(
|
|
TargetFC(paras['target_fc3w'], paras['target_fc3b']),
|
|
nn.Sigmoid(),
|
|
)
|
|
|
|
self.l4 = nn.Sequential(
|
|
TargetFC(paras['target_fc4w'], paras['target_fc4b']),
|
|
nn.Sigmoid(),
|
|
TargetFC(paras['target_fc5w'], paras['target_fc5b']),
|
|
)
|
|
|
|
def forward(self, x):
|
|
q = self.l1(x)
|
|
# q = F.dropout(q)
|
|
q = self.l2(q)
|
|
q = self.l3(q)
|
|
q = self.l4(q).squeeze()
|
|
return q
|
|
|
|
|
|
class TargetFC(nn.Module):
|
|
"""
|
|
Fully connection operations for target net
|
|
Note:
|
|
Weights & biases are different for different images in a batch,
|
|
thus here we use group convolution for calculating images in a batch with individual weights & biases.
|
|
"""
|
|
|
|
def __init__(self, weight, bias):
|
|
super(TargetFC, self).__init__()
|
|
self.weight = weight
|
|
self.bias = bias
|
|
|
|
def forward(self, input_):
|
|
|
|
input_re = input_.view(-1, input_.shape[0] * input_.shape[1], input_.shape[2], input_.shape[3])
|
|
weight_re = self.weight.view(self.weight.shape[0] * self.weight.shape[1], self.weight.shape[2],
|
|
self.weight.shape[3], self.weight.shape[4])
|
|
bias_re = self.bias.view(self.bias.shape[0] * self.bias.shape[1])
|
|
out = F.conv2d(input=input_re, weight=weight_re, bias=bias_re, groups=self.weight.shape[0])
|
|
|
|
return out.view(input_.shape[0], self.weight.shape[1], input_.shape[2], input_.shape[3])
|