import cv2 import sys import lpips import numpy as np from matplotlib import pyplot as plt import torch class LMDMeter: def __init__(self, backend='dlib', region='mouth'): self.backend = backend self.region = region # mouth or face if self.backend == 'dlib': import dlib # load checkpoint manually self.predictor_path = './shape_predictor_68_face_landmarks.dat' if not os.path.exists(self.predictor_path): raise FileNotFoundError('Please download dlib checkpoint from http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2') self.detector = dlib.get_frontal_face_detector() self.predictor = dlib.shape_predictor(self.predictor_path) else: import face_alignment try: self.predictor = face_alignment.FaceAlignment(face_alignment.LandmarksType._2D, flip_input=False) except: self.predictor = face_alignment.FaceAlignment(face_alignment.LandmarksType.TWO_D, flip_input=False) self.V = 0 self.N = 0 def get_landmarks(self, img): if self.backend == 'dlib': dets = self.detector(img, 1) for det in dets: shape = self.predictor(img, det) # ref: https://github.com/PyImageSearch/imutils/blob/c12f15391fcc945d0d644b85194b8c044a392e0a/imutils/face_utils/helpers.py lms = np.zeros((68, 2), dtype=np.int32) for i in range(0, 68): lms[i, 0] = shape.part(i).x lms[i, 1] = shape.part(i).y break else: lms = self.predictor.get_landmarks(img)[-1] # self.vis_landmarks(img, lms) lms = lms.astype(np.float32) return lms def vis_landmarks(self, img, lms): plt.imshow(img) plt.plot(lms[48:68, 0], lms[48:68, 1], marker='o', markersize=1, linestyle='-', lw=2) plt.show() def clear(self): self.V = 0 self.N = 0 def prepare_inputs(self, *inputs): outputs = [] for i, inp in enumerate(inputs): inp = inp.detach().cpu().numpy() inp = (inp * 255).astype(np.uint8) outputs.append(inp) return outputs def update(self, preds, truths): # assert B == 1 preds, truths = self.prepare_inputs(preds[0], truths[0]) # [H, W, 3] numpy array # get lms lms_pred = self.get_landmarks(preds) lms_truth = self.get_landmarks(truths) if self.region == 'mouth': lms_pred = lms_pred[48:68] lms_truth = lms_truth[48:68] # avarage lms_pred = lms_pred - lms_pred.mean(0) lms_truth = lms_truth - lms_truth.mean(0) # distance dist = np.sqrt(((lms_pred - lms_truth) ** 2).sum(1)).mean(0) self.V += dist self.N += 1 def measure(self): return self.V / self.N def write(self, writer, global_step, prefix=""): writer.add_scalar(os.path.join(prefix, f"LMD ({self.backend})"), self.measure(), global_step) def report(self): return f'LMD ({self.backend}) = {self.measure():.6f}' class PSNRMeter: def __init__(self): self.V = 0 self.N = 0 def clear(self): self.V = 0 self.N = 0 def prepare_inputs(self, *inputs): outputs = [] for i, inp in enumerate(inputs): if torch.is_tensor(inp): inp = inp.detach().cpu().numpy() outputs.append(inp) return outputs def update(self, preds, truths): preds, truths = self.prepare_inputs(preds, truths) # [B, N, 3] or [B, H, W, 3], range in [0, 1] # simplified since max_pixel_value is 1 here. psnr = -10 * np.log10(np.mean((preds - truths) ** 2)) self.V += psnr self.N += 1 def measure(self): return self.V / self.N def write(self, writer, global_step, prefix=""): writer.add_scalar(os.path.join(prefix, "PSNR"), self.measure(), global_step) def report(self): return f'PSNR = {self.measure():.6f}' class LPIPSMeter: def __init__(self, net='alex', device=None): self.V = 0 self.N = 0 self.net = net self.device = device if device is not None else torch.device('cuda' if torch.cuda.is_available() else 'cpu') self.fn = lpips.LPIPS(net=net).eval().to(self.device) def clear(self): self.V = 0 self.N = 0 def prepare_inputs(self, *inputs): outputs = [] for i, inp in enumerate(inputs): inp = inp.permute(0, 3, 1, 2).contiguous() # [B, 3, H, W] inp = inp.to(self.device) outputs.append(inp) return outputs def update(self, preds, truths): preds, truths = self.prepare_inputs(preds, truths) # [B, H, W, 3] --> [B, 3, H, W], range in [0, 1] v = self.fn(truths, preds, normalize=True).item() # normalize=True: [0, 1] to [-1, 1] self.V += v self.N += 1 def measure(self): return self.V / self.N def write(self, writer, global_step, prefix=""): writer.add_scalar(os.path.join(prefix, f"LPIPS ({self.net})"), self.measure(), global_step) def report(self): return f'LPIPS ({self.net}) = {self.measure():.6f}' lmd_meter = LMDMeter(backend='fan') psnr_meter = PSNRMeter() lpips_meter = LPIPSMeter() lmd_meter.clear() psnr_meter.clear() lpips_meter.clear() vid_path_1 = sys.argv[1] vid_path_2 = sys.argv[2] capture_1 = cv2.VideoCapture(vid_path_1) capture_2 = cv2.VideoCapture(vid_path_2) counter = 0 while True: ret_1, frame_1 = capture_1.read() ret_2, frame_2 = capture_2.read() if not ret_1 * ret_2: break # plt.imshow(frame_1[:, :, ::-1]) # plt.show() inp_1 = torch.FloatTensor(frame_1[..., ::-1] / 255.0)[None, ...].cuda() inp_2 = torch.FloatTensor(frame_2[..., ::-1] / 255.0)[None, ...].cuda() lmd_meter.update(inp_1, inp_2) psnr_meter.update(inp_1, inp_2) lpips_meter.update(inp_1, inp_2) counter+=1 if counter % 100 == 0: print(counter) print(lmd_meter.report()) print(psnr_meter.report()) print(lpips_meter.report())