dlib系列【4】imutils 获取脸部部位 及 扩展检测（如 虹膜检测、眨眼检测等）

imutils

imutils 是一个 基于 opencv 的 图像处理 工具包，对 opencv 进行了再次封装，简化了 opencv 的应用，这里不多赘述；

此外，imutils 专门配套 dlib 提供了处理 人脸的工具 face_utils；

dlib 获取人脸数据后，用特征点来描述五官，每个部位的 特征点 是固定的，想要进一步操作就需要处理这些 特征点，face_utils 就用于 快速处理这些 特征点

dlib 特征点索引

dlib  生成特征点对应下图，如 mouth：index 从 48 到 68，需要看这张图 才知道 哪些点对应嘴

face_utils 可直接获取

face_utils.FACIAL_LANDMARKS_IDXS
OrderedDict([('mouth', (48, 68)), ('inner_mouth', (60, 68)), ('right_eyebrow', (17, 22)), ('left_eyebrow', (22, 27)), ('right_eye', (36, 42)), ('left_eye', (42, 48)), ('nose', (27, 36)), ('jaw', (0, 17))])

dlib 特征点 存储

dlib 提取人脸的 68 个特征点后，用自身的 dlib.full_object_detection 格式保存他们的坐标信息，需要逐一通过 （shape.part(n).x， shape.part(n).y) 来获取


shape = predictor(img, d)        # 关键点检测
print(shape)        # <_dlib_pybind11.full_object_detection object at 0x0000013F450FCB20>
print(shape.part(0), shape.part(1), shape.part(2), shape.part(67))    # (322, 217) (319, 238) (319, 260) (386, 321)

face_utils 可直接转换

def shape_to_np(shape, dtype="int"):
    # initialize the list of (x, y)-coordinates
    coords = np.zeros((shape.num_parts, 2), dtype=dtype)

    # loop over all facial landmarks and convert them
    # to a 2-tuple of (x, y)-coordinates
    for i in range(0, shape.num_parts):
        coords[i] = (shape.part(i).x, shape.part(i).y)

    # return the list of (x, y)-coordinates
    return coords

当然，其实没做什么大事，只是把 坐标 放到 （68，2）的矩阵中

标记人脸部位

获取人脸部位后，face_utils 可直接对 这些部位进行标记

def visualize_facial_landmarks(image, shape, colors=None, alpha=0.75):
    # create two copies of the input image -- one for the
    # overlay and one for the final output image
    overlay = image.copy()
    output = image.copy()

    # if the colors list is None, initialize it with a unique
    # color for each facial landmark region
    if colors is None:
        colors = [(19, 199, 109), (79, 76, 240), (230, 159, 23),
            (168, 100, 168), (158, 163, 32),
            (163, 38, 32), (180, 42, 220)]

    # loop over the facial landmark regions individually
    for (i, name) in enumerate(FACIAL_LANDMARKS_IDXS.keys()):
        # grab the (x, y)-coordinates associated with the
        # face landmark
        (j, k) = FACIAL_LANDMARKS_IDXS[name]
        pts = shape[j:k]

        # check if are supposed to draw the jawline
        if name == "jaw":
            # since the jawline is a non-enclosed facial region,
            # just draw lines between the (x, y)-coordinates
            for l in range(1, len(pts)):
                ptA = tuple(pts[l - 1])
                ptB = tuple(pts[l])
                cv2.line(overlay, ptA, ptB, colors[i], 2)

        # otherwise, compute the convex hull of the facial
        # landmark coordinates points and display it
        else:
            hull = cv2.convexHull(pts)
            cv2.drawContours(overlay, [hull], -1, colors[i], -1)

    # apply the transparent overlay
    cv2.addWeighted(overlay, alpha, output, 1 - alpha, 0, output)

    # return the output image
    return output

示例代码

import cv2
import dlib
from imutils import face_utils

detector = dlib.get_frontal_face_detector()
predictor_path = 'shape_predictor_68_face_landmarks.dat'
predictor = dlib.shape_predictor(predictor_path)

img = cv2.imread('img/cl.jpg')
det = detector(img, 1)
shape = predictor(img, det[0])
print(shape)

shape = face_utils.shape_to_np(shape)
img = face_utils.visualize_facial_landmarks(img, shape)
cv2.imshow('img', img)
cv2.waitKey(0)

输出

虹膜检测

检测流程：

执行面部关键点检测。

使用相应的关键点为两只眼睛创建蒙版。

阈值-。这一步取决于对象的虹膜色调。例如，如果你试图检测蓝眼睛人的虹膜，则应使用图像的红色或绿色通道来增加与白色背景的对比度。

找到二值化虹膜图像的质心。

找到二值化虹膜图像的轮廓。

找到所有虹膜轮廓的最小封闭圆。

比较步骤4中获得的质心与所有轮廓的最小封闭圆质心之间的距离。

选择到质心距离最小的圆。

使用步骤8中获得的圆创建蒙版和反向蒙版。

对整个图像应用颜色变换，并将该图像与虹膜蒙版相乘。

将原始图像乘以反向蒙版。

将步骤10和11的结果图像添加到一起。

import cv2
import dlib
import numpy as np
from imutils import face_utils

im = cv2.imread("img/cl.jpg")

# 检测脸部关键点
PREDICTOR_PATH = "shape_predictor_68_face_landmarks.dat"
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor(PREDICTOR_PATH)
det = detector(im, 1)
shape = predictor(im, det[0])
shape = face_utils.shape_to_np(shape)

def createEyeMask(eyeLandmarks, im):
    leftEyePoints = eyeLandmarks
    eyeMask = np.zeros_like(im)
    cv2.fillConvexPoly(eyeMask, np.int32(leftEyePoints), (255, 255, 255))
    eyeMask = np.uint8(eyeMask)
    return eyeMask

def findIris(eyeMask, im, thresh):
    r = im[:, :, 2]
    _, binaryIm = cv2.threshold(r, thresh, 255, cv2.THRESH_BINARY_INV)
    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (4, 4))
    morph = cv2.dilate(binaryIm, kernel, 1)
    morph = cv2.merge((morph, morph, morph))
    morph = morph.astype(float) / 255
    eyeMask = eyeMask.astype(float) / 255
    iris = cv2.multiply(eyeMask, morph)
    return iris

def findCentroid(iris):
    M = cv2.moments(iris[:, :, 0])
    cX = int(M["m10"] / M["m00"])
    cY = int(M["m01"] / M["m00"])
    centroid = (cX, cY)
    return centroid

def createIrisMask(iris, centroid):
    cnts, _ = cv2.findContours(np.uint8(iris[:, :, 0]), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    flag = 10000
    final_cnt = None
    for cnt in cnts:
        (x, y), radius = cv2.minEnclosingCircle(cnt)
        distance = abs(centroid[0] - x) + abs(centroid[1] - y)
        if distance < flag:
            flag = distance
            final_cnt = cnt
        else:
            continue
    (x, y), radius = cv2.minEnclosingCircle(final_cnt)
    center = (int(x), int(y))
    radius = int(radius) - 2

    irisMask = np.zeros_like(iris)
    inverseIrisMask = np.ones_like(iris) * 255
    cv2.circle(irisMask, center, radius, (255, 255, 255), -1)
    cv2.circle(inverseIrisMask, center, radius, (0, 0, 0), -1)
    irisMask = cv2.GaussianBlur(irisMask, (5, 5), cv2.BORDER_DEFAULT)
    inverseIrisMask = cv2.GaussianBlur(inverseIrisMask, (5, 5), cv2.BORDER_DEFAULT)

    return irisMask, inverseIrisMask

def changeEyeColor(im, irisMask, inverseIrisMask):
    imCopy = cv2.applyColorMap(im, cv2.COLORMAP_TWILIGHT_SHIFTED)
    imCopy = imCopy.astype(float) / 255
    irisMask = irisMask.astype(float) / 255
    inverseIrisMask = inverseIrisMask.astype(float) / 255
    im = im.astype(float) / 255
    faceWithoutEye = cv2.multiply(inverseIrisMask, im)
    newIris = cv2.multiply(irisMask, imCopy)
    result = faceWithoutEye + newIris
    return result

def float642Uint8(im):
    im2Convert = im.astype(np.float64) / np.amax(im)
    im2Convert = 255 * im2Convert
    convertedIm = im2Convert.astype(np.uint8)
    return convertedIm

# 创建眼睛蒙版
leftEyeMask = createEyeMask(shape[36:42], im)
rightEyeMask = createEyeMask(shape[43:49], im)

# 设定阈值来找到虹膜
leftIris = findIris(leftEyeMask, im, 40)
rightIris = findIris(rightEyeMask, im, 50)

# 寻找质心
leftIrisCentroid = findCentroid(leftIris)
rightIrisCentroid = findCentroid(rightIris)

# 生成虹膜蒙版及其反蒙版
leftIrisMask, leftInverseIrisMask = createIrisMask(leftIris, leftIrisCentroid)
rightIrisMask, rightInverseIrisMask = createIrisMask(rightIris, rightIrisCentroid)

# 改变眼睛的颜色，并合并到原始图像
coloredEyesLady = changeEyeColor(im, rightIrisMask, rightInverseIrisMask)
coloredEyesLady = float642Uint8(coloredEyesLady)
coloredEyesLady = changeEyeColor(coloredEyesLady, leftIrisMask, leftInverseIrisMask)

# 现在的结果
cv2.imshow("", coloredEyesLady)
cv2.waitKey(0)

输出

眼睛检测

眨眼检测

疲劳检测

思路都大致相同，检测到特征点，然后提取某个部位，进行后续操作和计算，具体见参考资料，有些代码跑不通，懒得调了

参考资料：

https://steven-cloud.blog.csdn.net/article/details/107105690?spm=1001.2101.3001.6661.1&utm_medium=distribute.pc_relevant_t0.none-task-blog-2%7Edefault%7ECTRLIST%7ETopBlog-1-107105690-blog-114839030.topblog&depth_1-utm_source=distribute.pc_relevant_t0.none-task-blog-2%7Edefault%7ECTRLIST%7ETopBlog-1-107105690-blog-114839030.topblog&utm_relevant_index=1　　imutils库源码解析，看它如何调用opencv（二）- 面部工具

https://blog.csdn.net/woshicver/article/details/121987470　　　　使用OpenCV+Dlib检测虹膜并改变其颜色

https://blog.csdn.net/qq_38641985/article/details/115755417　　 基于dlib的眼睛检测

https://www.it610.com/article/1303927614737518592.htm　　Dlib模型之驾驶员疲劳检测一（眨眼）

https://zhuanlan.zhihu.com/p/498712772　　　　　　　　　 人脸检测实战高级：使用 OpenCV、Python 和 dlib 完成眨眼检测

https://www.cnblogs.com/gmhappy/p/11864119.html　　　　 dlib 基于摄像流检测眨眼次数

https://blog.csdn.net/wanlong_peng/article/details/115841226　　openCV和python基于dlib库实现眨眼/睁闭眼检测--亲测高效