face detect and classify from image

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 coneypo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.rst

@@ -11,13 +11,13 @@ Detect and recognize single/multi-faces from camera;
 
 #. 摄像头人脸录入 / Face register
 
-   .. image:: introduction/get_face_from_camera.png
+   .. image:: introduction/face_register.png
       :align: center
 
    请不要离摄像头过近，人脸超出摄像头范围时会有 "OUT OF RANGE" 提醒 /
    Please do not be too close to the camera, or you can't save faces with "OUT OF RANGE" warning;
 
-   .. image:: introduction/get_face_from_camera_out_of_range.png
+   .. image:: introduction/face_register_warning.png
       :align: center
 
 #. 提取特征建立人脸数据库 / Generate database from images captured
@@ -25,25 +25,29 @@ Detect and recognize single/multi-faces from camera;
    
    当单张人脸 / When single-face:
    
-   .. image:: introduction/face_reco_single_person.png
+   .. image:: introduction/face_reco_single.png
+      :align: center
+
+   利用 OT 对于单张人脸追踪/ Use OT to track, which can improve FPS from 1.x to 20.x:
+
+   .. image:: introduction/face_reco_single_ot.png
       :align: center
 
    当多张人脸 / When multi-faces:
 
-   一张已录入人脸 + 未录入 unknown 人脸 / 1x known face + 2x unknown face:
+   .. image:: introduction/face_reco_multi.png
+      :align: center
+   
+   利用 OT 来实现 / When multi-faces with OT:
 
-   .. image:: introduction/face_reco_multi_people.png
+   .. image:: introduction/face_reco_multi_ot.png
       :align: center
 
-   同时识别多张已录入人脸 / Multi-faces recognition at the same time:
+   定制显示名字, 可以写中文 / Customize names:
 
-   .. image:: introduction/face_reco_two_people_in_database.png
+   .. image:: introduction/face_reco_with_name.png
       :align: center
 
-   实时人脸特征描述子计算 / Real-time face descriptor computation:
-
-   .. image:: introduction/face_descriptor_single_person.png
-      :align: center
 
 ** 关于精度 / About accuracy:
 
@@ -58,11 +62,18 @@ Detect and recognize single/multi-faces from camera;
 Overview
 ********
 
-此项目中人脸识别的实现流程 / The design of this repo:
+此项目中人脸识别的实现流程 (no OT, 每一帧都进行检测+识别) / The design of this repo:
 
 .. image:: introduction/overview.png
    :align: center
 
+实现流程(with OT, 初始帧进行检测+识别，后续帧检测+质心跟踪) / The design of this repo:
+
+.. image:: introduction/overview_with_ot.png
+   :align: center
+
+如果利用 OT 来跟踪，可以大大提高 FPS, 因为做识别时候需要提取特征描述子的耗时很多;
+
 Steps
 *****
 
@@ -98,6 +109,12 @@ Steps
 
       python3 face_reco_from_camera.py
 
+#. 或者利用 OT 算法，调用摄像头进行实时人脸识别/ Real-time face recognition with OT
+
+   .. code-block:: bash
+
+      python3 face_reco_from_camera_ot_single_person.py
+      python3 face_reco_from_camera_ot_multi_people.py
 
 About Source Code
 *****************
@@ -107,34 +124,27 @@ Repo 的 tree / 树状图:
 ::
 
     .
-    ├── get_faces_from_camera.py        # Step1. Face register
-    ├── features_extraction_to_csv.py   # Step2. Feature extraction
-    ├── face_reco_from_camera.py        # Step3. Face recognizer
-    ├── face_descriptor_from_camera.py  # Face descriptor computation
-    ├── how_to_use_camera.py            # Use the default camera by opencv
+    ├── get_faces_from_camera.py        		# Step 1. Face register
+    ├── features_extraction_to_csv.py   		# Step 2. Feature extraction
+    ├── face_reco_from_camera.py        		# Step 3. Face recognizer
+    ├── face_reco_from_camera_ot_single_person.py       # Step 3. Face recognizer with OT for single person
+    ├── face_reco_from_camera_ot_multi_people.py        # Step 3. Face recognizer with OT for multi people
+    ├── face_descriptor_from_camera.py  		# Face descriptor computation
+    ├── how_to_use_camera.py            		# Use the default camera by opencv
     ├── data
-    │   ├── data_dlib                   # Dlib's model
+    │   ├── data_dlib        			        # Dlib's model
     │   │   ├── dlib_face_recognition_resnet_model_v1.dat
     │   │   └── shape_predictor_68_face_landmarks.dat
-    │   ├── data_faces_from_camera      # Face images captured from camera (will generate after step 1)
+    │   ├── data_faces_from_camera      		# Face images captured from camera (will generate after step 1)
     │   │   ├── person_1
     │   │   │   ├── img_face_1.jpg
     │   │   │   └── img_face_2.jpg
     │   │   └── person_2
     │   │       └── img_face_1.jpg
     │   │       └── img_face_2.jpg
-    │   └── features_all.csv            # CSV to save all the features of known faces (will generate after step 2)
-    ├── introduction                    # Some files for readme.rst
-    │   ├── Dlib_Face_recognition_by_coneypo.pptx
-    │   ├── face_reco_single_person_customize_name.png
-    │   ├── face_reco_single_person.png
-    │   ├── face_reco_two_people_in_database.png
-    │   ├── face_reco_two_people.png
-    │   ├── get_face_from_camera_out_of_range.png
-    │   ├── get_face_from_camera.png
-    │   └── overview.png
+    │   └── features_all.csv            		# CSV to save all the features of known faces (will generate after step 2)
     ├── README.rst
-    └── requirements.txt                # Some python packages needed
+    └── requirements.txt                		# Some python packages needed
 
 用到的 Dlib 相关模型函数:
 
@@ -196,6 +206,11 @@ Python 源码介绍如下:
    
    * 将捕获到的人脸数据和之前存的人脸数据进行对比计算欧式距离, 由此判断是否是同一个人;
 
+#. face_reco_from_camera_ot_single_person/multi_people.py:
+	
+   区别于 face_reco_from_camera.py (对每一帧都进行检测+识别)，只会对初始帧做检测+识别，对后续帧做检测+质心跟踪;
+   
+
 #. (optional) face_descriptor_from_camera.py
 
    调用摄像头进行实时特征描述子计算; / Real-time face descriptor computation;
@@ -213,17 +228,15 @@ Tips:
 
 #. 人脸录入的时候先建文件夹再保存图片, 先 ``N`` 再 ``S`` / Press ``N`` before ``S``
 
-#. 关于人脸识别卡顿 FPS 低问题, 不做 compare 的时候, 光跑 face_descriptor_from_camera.py 中 face_reco_model.compute_face_descriptor
-   在 CPU: i7-8700K FPS: 5~6, 所以主要提取特征时候耗资源
+#. 关于人脸识别卡顿 FPS 低问题, 原因是特征描述子提取很费时间, 光跑 face_descriptor_from_camera.py 中 face_reco_model.compute_face_descriptor 在 CPU: i7-8700K 得到的最终 FPS: 5~6 (检测在 0.03s, 特征描述子提取在 0.158s, 和已知人脸进行遍历对比在 0.003s 左右), 所以主要提取特征时候耗资源, 可以用 OT 去做追踪，而不是对每一帧都做检测+识别
 
 可以访问我的博客获取本项目的更详细介绍，如有问题可以邮件联系我 /
 For more details, please refer to my blog (in chinese) or mail to me :
 
 * Blog: https://www.cnblogs.com/AdaminXie/p/9010298.html
+
+* 关于 OT 部分的更新在 Blog: https://www.cnblogs.com/AdaminXie/p/13566269.html
   
 * Mail: coneypo@foxmail.com ( Dlib 相关 repo 问题请联系 @foxmail 而不是 @intel )
 
-
-仅限于交流学习, 商业合作勿扰;
-
-Thanks for your support.
+Thanks for your support.

face_reco_from_camera.py

@@ -1,58 +1,50 @@
-# 摄像头实时人脸识别
-# Real-time face recognition
+# Copyright (C) 2020 coneypo
+# SPDX-License-Identifier: MIT
 
 # Author:   coneypo
 # Blog:     http://www.cnblogs.com/AdaminXie
 # GitHub:   https://github.com/coneypo/Dlib_face_recognition_from_camera
+# Mail:     coneypo@foxmail.com
 
-# Created at 2018-05-11
-# Updated at 2020-05-29
+# 人脸识别 / Real-time face detection and recognition from images
 
-import dlib         # 人脸处理的库 Dlib
-import numpy as np  # 数据处理的库 Numpy
-import cv2          # 图像处理的库 OpenCV
-import pandas as pd # 数据处理的库 Pandas
+import dlib
+import numpy as np
+import cv2
+import pandas as pd
 import os
 import time
 from PIL import Image, ImageDraw, ImageFont
 
-# 1. Dlib 正向人脸检测器
+# Dlib 正向人脸检测器 / Use frontal face detector of Dlib
 detector = dlib.get_frontal_face_detector()
 
-# 2. Dlib 人脸 landmark 特征点检测器
+# Dlib 人脸 landmark 特征点检测器 / Get face landmarks
 predictor = dlib.shape_predictor('data/data_dlib/shape_predictor_68_face_landmarks.dat')
 
-# 3. Dlib Resnet 人脸识别模型，提取 128D 的特征矢量
+# Dlib Resnet 人脸识别模型，提取 128D 的特征矢量 / Use Dlib resnet50 model to get 128D face descriptor
 face_reco_model = dlib.face_recognition_model_v1("data/data_dlib/dlib_face_recognition_resnet_model_v1.dat")
 
 
 class Face_Recognizer:
     def __init__(self):
-        # 用来存放所有录入人脸特征的数组 / Save the features of faces in the database
-        self.features_known_list = []
+        self.feature_known_list = []                # 用来存放所有录入人脸特征的数组 / Save the features of faces in the database
+        self.name_known_list = []                   # 存储录入人脸名字 / Save the name of faces in the database
 
-        # 存储录入人脸名字 / Save the name of faces known
-        self.name_known_cnt = 0
-        self.name_known_list = []
-
-        # 存储当前摄像头中捕获到的所有人脸的坐标名字 / Save the positions and names of current faces captured
-        self.pos_camera_list = []
-        self.name_camera_list = []
-        # 存储当前摄像头中捕获到的人脸数
-        self.faces_cnt = 0
-        # 存储当前摄像头中捕获到的人脸特征
-        self.features_camera_list = []
+        self.current_frame_face_cnt = 0             # 存储当前摄像头中捕获到的人脸数 / Counter for faces in current frame
+        self.current_frame_feature_list = []        # 存储当前摄像头中捕获到的人脸特征 / Features of faces in current frame
+        self.current_frame_name_position_list = []  # 存储当前摄像头中捕获到的所有人脸的名字坐标 / Positions of faces in current frame
+        self.current_frame_name_list = []           # 存储当前摄像头中捕获到的所有人脸的名字 / Names of faces in current frame
 
         # Update FPS
         self.fps = 0
         self.frame_start_time = 0
 
-    # 从 "features_all.csv" 读取录入人脸特征
+    # 从 "features_all.csv" 读取录入人脸特征 / Get known faces from "features_all.csv"
     def get_face_database(self):
         if os.path.exists("data/features_all.csv"):
             path_features_known_csv = "data/features_all.csv"
             csv_rd = pd.read_csv(path_features_known_csv, header=None)
-            # 2. 读取已知人脸数据 / Print known faces
             for i in range(csv_rd.shape[0]):
                 features_someone_arr = []
                 for j in range(0, 128):
@@ -60,10 +52,9 @@ class Face_Recognizer:
                         features_someone_arr.append('0')
                     else:
                         features_someone_arr.append(csv_rd.iloc[i][j])
-                self.features_known_list.append(features_someone_arr)
+                self.feature_known_list.append(features_someone_arr)
                 self.name_known_list.append("Person_"+str(i+1))
-            self.name_known_cnt = len(self.name_known_list)
-            print("Faces in Database：", len(self.features_known_list))
+            print("Faces in Database：", len(self.feature_known_list))
             return 1
         else:
             print('##### Warning #####', '\n')
@@ -94,7 +85,7 @@ class Face_Recognizer:
 
         cv2.putText(img_rd, "Face Recognizer", (20, 40), font, 1, (255, 255, 255), 1, cv2.LINE_AA)
         cv2.putText(img_rd, "FPS:   " + str(self.fps.__round__(2)), (20, 100), font, 0.8, (0, 255, 0), 1, cv2.LINE_AA)
-        cv2.putText(img_rd, "Faces: " + str(self.faces_cnt), (20, 140), font, 0.8, (0, 255, 0), 1, cv2.LINE_AA)
+        cv2.putText(img_rd, "Faces: " + str(self.current_frame_face_cnt), (20, 140), font, 0.8, (0, 255, 0), 1, cv2.LINE_AA)
         cv2.putText(img_rd, "Q: Quit", (20, 450), font, 0.8, (255, 255, 255), 1, cv2.LINE_AA)
 
     def draw_name(self, img_rd):
@@ -102,120 +93,93 @@ class Face_Recognizer:
         font = ImageFont.truetype("simsun.ttc", 30)
         img = Image.fromarray(cv2.cvtColor(img_rd, cv2.COLOR_BGR2RGB))
         draw = ImageDraw.Draw(img)
-        for i in range(self.faces_cnt):
-            # cv2.putText(img_rd, self.name_camera_list[i], self.pos_camera_list[i], font, 0.8, (0, 255, 255), 1, cv2.LINE_AA)
-            draw.text(xy=self.pos_camera_list[i], text=self.name_camera_list[i], font=font)
+        for i in range(self.current_frame_face_cnt):
+            # cv2.putText(img_rd, self.current_frame_name_list[i], self.current_frame_name_position_list[i], font, 0.8, (0, 255, 255), 1, cv2.LINE_AA)
+            draw.text(xy=self.current_frame_name_position_list[i], text=self.current_frame_name_list[i], font=font)
             img_with_name = cv2.cvtColor(np.array(img), cv2.COLOR_RGB2BGR)
         return img_with_name
 
-    # 修改显示人名
-    def modify_name_camera_list(self):
+    # 修改显示人名 / Show names in chinese
+    def show_chinese_name(self):
         # Default known name: person_1, person_2, person_3
-        self.name_known_list[0] ='张三'.encode('utf-8').decode()
-        self.name_known_list[1] ='李四'.encode('utf-8').decode()
+        if self.current_frame_face_cnt >= 1:
+            self.name_known_list[0] ='张三'.encode('utf-8').decode()
+        # self.name_known_list[1] ='李四'.encode('utf-8').decode()
         # self.name_known_list[2] ='xx'.encode('utf-8').decode()
         # self.name_known_list[3] ='xx'.encode('utf-8').decode()
         # self.name_known_list[4] ='xx'.encode('utf-8').decode()
 
-    # 处理获取的视频流，进行人脸识别 / Input video stream and face reco process
-    def process(self, stream):
-        # 1. 读取存放所有人脸特征的 csv
+    # 处理获取的视频流，进行人脸识别 / Face detection and recognition from input video stream
+    def process(self):
+        # 1. 读取存放所有人脸特征的 csv / Get faces known from "features.all.csv"
         if self.get_face_database():
-            while stream.isOpened():
-                flag, img_rd = stream.read()
-                faces = detector(img_rd, 0)
-                kk = cv2.waitKey(1)
-                # 按下 q 键退出 / Press 'q' to quit
-                if kk == ord('q'):
-                    break
-                else:
-                    self.draw_note(img_rd)
-                    self.features_camera_list = []
-                    self.faces_cnt = 0
-                    self.pos_camera_list = []
-                    self.name_camera_list = []
 
-                    # 2. 检测到人脸 / when face detected
-                    if len(faces) != 0:
-                        # 3. 获取当前捕获到的图像的所有人脸的特征，存储到 self.features_camera_list
-                        # 3. Get the features captured and save into self.features_camera_list
-                        for i in range(len(faces)):
-                            shape = predictor(img_rd, faces[i])
-                            self.features_camera_list.append(face_reco_model.compute_face_descriptor(img_rd, shape))
+            print(">>> Frame start")
+            img_rd = cv2.imread("data/data_faces_for_test/test_faces_1.jpg")
+            faces = detector(img_rd, 1)
+            self.draw_note(img_rd)
 
-                        # 4. 遍历捕获到的图像中所有的人脸 / Traversal all the faces in the database
-                        for k in range(len(faces)):
-                            print("##### camera person", k + 1, "#####")
-                            # 让人名跟随在矩形框的下方
-                            # 确定人名的位置坐标
-                            # 先默认所有人不认识，是 unknown
-                            # Set the default names of faces with "unknown"
-                            self.name_camera_list.append("unknown")
+            # 2. 检测到人脸 / Face detected in current frame
+            if len(faces) != 0:
+                # 3. 获取当前捕获到的图像的所有人脸的特征 / Compute the face descriptors for faces in current frame
+                for i in range(len(faces)):
+                    shape = predictor(img_rd, faces[i])
+                    self.current_frame_feature_list.append(face_reco_model.compute_face_descriptor(img_rd, shape))
+                # 4. 遍历捕获到的图像中所有的人脸 / Traversal all the faces in the database
+                for k in range(len(faces)):
+                    print(">>>>>> For face", k+1, " in camera")
+                    # 先默认所有人不认识，是 unknown / Set the default names of faces with "unknown"
+                    self.current_frame_name_list.append("unknown")
 
-                            # 每个捕获人脸的名字坐标 / Positions of faces captured
-                            self.pos_camera_list.append(tuple(
-                                [faces[k].left(), int(faces[k].bottom() + (faces[k].bottom() - faces[k].top()) / 4)]))
+                    # 每个捕获人脸的名字坐标 / Positions of faces captured
+                    self.current_frame_name_position_list.append(tuple(
+                        [faces[k].left(), int(faces[k].bottom() + (faces[k].bottom() - faces[k].top()) / 4)]))
 
-                            # 5. 对于某张人脸，遍历所有存储的人脸特征
-                            # For every faces detected, compare the faces in the database
-                            e_distance_list = []
-                            for i in range(len(self.features_known_list)):
-                                # 如果 person_X 数据不为空
-                                if str(self.features_known_list[i][0]) != '0.0':
-                                    print("with person", str(i + 1), "the e distance: ", end='')
-                                    e_distance_tmp = self.return_euclidean_distance(self.features_camera_list[k],
-                                                                                    self.features_known_list[i])
-                                    print(e_distance_tmp)
-                                    e_distance_list.append(e_distance_tmp)
-                                else:
-                                    # 空数据 person_X
-                                    e_distance_list.append(999999999)
-                            # 6. 寻找出最小的欧式距离匹配 / Find the one with minimum e distance
-                            similar_person_num = e_distance_list.index(min(e_distance_list))
-                            print("Minimum e distance with person", self.name_known_list[similar_person_num])
+                    # 5. 对于某张人脸，遍历所有存储的人脸特征
+                    # For every faces detected, compare the faces in the database
+                    current_frame_e_distance_list = []
+                    for i in range(len(self.feature_known_list)):
+                        # 如果 person_X 数据不为空
+                        if str(self.feature_known_list[i][0]) != '0.0':
+                            print("   >>> With person", str(i + 1), ", the e distance: ", end='')
+                            e_distance_tmp = self.return_euclidean_distance(self.current_frame_feature_list[k],
+                                                                            self.feature_known_list[i])
+                            print(e_distance_tmp)
+                            current_frame_e_distance_list.append(e_distance_tmp)
+                        else:
+                            # 空数据 person_X
+                            current_frame_e_distance_list.append(999999999)
+                    # 6. 寻找出最小的欧式距离匹配 / Find the one with minimum e distance
+                    similar_person_num = current_frame_e_distance_list.index(min(current_frame_e_distance_list))
+                    print("   >>> Minimum e distance with ", self.name_known_list[similar_person_num], ": ", min(current_frame_e_distance_list))
 
-                            if min(e_distance_list) < 0.4:
-                                self.name_camera_list[k] = self.name_known_list[similar_person_num]
-                                print("May be person " + str(self.name_known_list[similar_person_num]))
-                            else:
-                                print("Unknown person")
-
-                            # 矩形框 / Draw rectangle
-                            for kk, d in enumerate(faces):
-                                # 绘制矩形框
-                                cv2.rectangle(img_rd, tuple([d.left(), d.top()]), tuple([d.right(), d.bottom()]),
-                                              (0, 255, 255), 2)
-                            print('\n')
-
-                        self.faces_cnt = len(faces)
-                        # 7. 在这里更改显示的人名 / Modify name if needed
-                        self.modify_name_camera_list()
-                        # 8. 写名字 / Draw name
-                        # self.draw_name(img_rd)
-                        img_with_name = self.draw_name(img_rd)
+                    if min(current_frame_e_distance_list) < 0.4:
+                        self.current_frame_name_list[k] = self.name_known_list[similar_person_num]
+                        print("   >>> Face recognition result:  " + str(self.name_known_list[similar_person_num]))
                     else:
-                        img_with_name = img_rd
+                        print("   >>> Face recognition result: Unknown person")
 
-                print("Faces in camera now:", self.name_camera_list, "\n")
+                    # 矩形框 / Draw rectangle
+                    for kk, d in enumerate(faces):
+                        # 绘制矩形框
+                        cv2.rectangle(img_rd, tuple([d.left(), d.top()]), tuple([d.right(), d.bottom()]),
+                                      (0, 255, 255), 2)
 
-                cv2.imshow("camera", img_with_name)
+                self.current_frame_face_cnt = len(faces)
 
-                # 9. 更新 FPS / Update stream FPS
-                self.update_fps()
+                img_rd = self.draw_name(img_rd)
 
-    # OpenCV 调用摄像头并进行 process
-    def run(self):
-        cap = cv2.VideoCapture(0)
-        cap.set(3, 480)
-        self.process(cap)
+                print(">>>>>> Faces in camera now:", self.current_frame_name_list)
 
-        cap.release()
-        cv2.destroyAllWindows()
+                cv2.imshow("camera", img_rd)
+                cv2.waitKey(0)
+
+                print(">>> Frame ends\n\n")
 
 
 def main():
     Face_Recognizer_con = Face_Recognizer()
-    Face_Recognizer_con.run()
+    Face_Recognizer_con.process()
 
 
 if __name__ == '__main__':

face_reco_from_camera_ot_multi_people.py

@@ -0,0 +1,285 @@
+# Copyright (C) 2020 coneypo
+# SPDX-License-Identifier: MIT
+
+# Author:   coneypo
+# Blog:     http://www.cnblogs.com/AdaminXie
+# GitHub:   https://github.com/coneypo/Dlib_face_recognition_from_camera
+# Mail:     coneypo@foxmail.com
+
+# 利用 OT 人脸追踪, 实时人脸识别 / Real-time face detection and recognition via OT for single face
+
+import dlib
+import numpy as np
+import cv2
+import os
+import pandas as pd
+import time
+
+# Dlib 正向人脸检测器 / Use frontal face detector of Dlib
+detector = dlib.get_frontal_face_detector()
+
+# Dlib 人脸 landmark 特征点检测器 / Get face landmarks
+predictor = dlib.shape_predictor('data/data_dlib/shape_predictor_68_face_landmarks.dat')
+
+# Dlib Resnet 人脸识别模型，提取 128D 的特征矢量 / Use Dlib resnet50 model to get 128D face descriptor
+face_reco_model = dlib.face_recognition_model_v1("data/data_dlib/dlib_face_recognition_resnet_model_v1.dat")
+
+
+class Face_Recognizer:
+    def __init__(self):
+        self.font = cv2.FONT_ITALIC
+
+        # For FPS
+        self.frame_time = 0
+        self.frame_start_time = 0
+        self.fps = 0
+
+        # cnt for frame
+        self.frame_cnt = 0
+
+        # 用来存放所有录入人脸特征的数组 / Save the features of faces in the database
+        self.features_known_list = []
+        # 存储录入人脸名字 / Save the name of faces in the database
+        self.name_known_list = []
+
+        # 用来存储上一帧和当前帧 ROI 的质心坐标 / List to save centroid positions of ROI in frame N-1 and N
+        self.last_frame_centroid_list = []
+        self.current_frame_centroid_list = []
+
+        # 用来存储上一帧和当前帧检测出目标的名字 / List to save names of objects in frame N-1 and N
+        self.last_frame_names_list = []
+        self.current_frame_face_name_list = []
+
+        # 上一帧和当前帧中人脸数的计数器 / cnt for faces in frame N-1 and N
+        self.last_frame_face_cnt = 0
+        self.current_frame_face_cnt = 0
+
+        # 用来存放进行识别时候对比的欧氏距离 / Save the e-distance for faceX when recognizing
+        self.current_frame_face_X_e_distance_list = []
+
+        # 存储当前摄像头中捕获到的所有人脸的坐标名字 / Save the positions and names of current faces captured
+        self.current_frame_face_position_list = []
+        # 存储当前摄像头中捕获到的人脸特征 / Save the features of people in current frame
+        self.current_frame_face_features_list = []
+
+        # e distance between centroid of ROI in last and current frame
+        self.last_current_frame_centroid_e_distance = 0
+
+    # 从 "features_all.csv" 读取录入人脸特征 / Get known faces from "features_all.csv"
+    def get_face_database(self):
+        if os.path.exists("data/features_all.csv"):
+            path_features_known_csv = "data/features_all.csv"
+            csv_rd = pd.read_csv(path_features_known_csv, header=None)
+            for i in range(csv_rd.shape[0]):
+                features_someone_arr = []
+                for j in range(0, 128):
+                    if csv_rd.iloc[i][j] == '':
+                        features_someone_arr.append('0')
+                    else:
+                        features_someone_arr.append(csv_rd.iloc[i][j])
+                self.features_known_list.append(features_someone_arr)
+                self.name_known_list.append("Person_" + str(i + 1))
+            print("Faces in Database：", len(self.features_known_list))
+            return 1
+        else:
+            print('##### Warning #####', '\n')
+            print("'features_all.csv' not found!")
+            print(
+                "Please run 'get_faces_from_camera.py' and 'features_extraction_to_csv.py' before 'face_reco_from_camera.py'",
+                '\n')
+            print('##### End Warning #####')
+            return 0
+
+    # 获取处理之后 stream 的帧数 / Get the fps of video stream
+    def update_fps(self):
+        now = time.time()
+        self.frame_time = now - self.frame_start_time
+        self.fps = 1.0 / self.frame_time
+        self.frame_start_time = now
+        # 计算两个128D向量间的欧式距离 / Compute the e-distance between two 128D features
+
+    @staticmethod
+    def return_euclidean_distance(feature_1, feature_2):
+        feature_1 = np.array(feature_1)
+        feature_2 = np.array(feature_2)
+        dist = np.sqrt(np.sum(np.square(feature_1 - feature_2)))
+        return dist
+
+    # / Use centroid tracker to link face_x in current frame with person_x in last frame
+    def centroid_tracker(self):
+        for i in range(len(self.current_frame_centroid_list)):
+            e_distance_current_frame_person_x_list = []
+            # For object 1 in current_frame, compute e-distance with object 1/2/3/4/... in last frame
+            for j in range(len(self.last_frame_centroid_list)):
+                self.last_current_frame_centroid_e_distance = self.return_euclidean_distance(
+                    self.current_frame_centroid_list[i], self.last_frame_centroid_list[j])
+
+                e_distance_current_frame_person_x_list.append(
+                    self.last_current_frame_centroid_e_distance)
+
+            last_frame_num = e_distance_current_frame_person_x_list.index(
+                min(e_distance_current_frame_person_x_list))
+            self.current_frame_face_name_list[i] = self.last_frame_face_name_list[last_frame_num]
+
+    # 生成的 cv2 window 上面添加说明文字 / putText on cv2 window
+    def draw_note(self, img_rd):
+        # 添加说明 / Add some statements
+        cv2.putText(img_rd, "Face recognizer with OT", (20, 40), self.font, 1, (255, 255, 255), 1, cv2.LINE_AA)
+        cv2.putText(img_rd, "FPS:    " + str(self.fps.__round__(2)), (20, 100), self.font, 0.8, (0, 255, 0), 1,
+                    cv2.LINE_AA)
+        cv2.putText(img_rd, "Faces:  " + str(self.current_frame_face_cnt), (20, 130), self.font, 0.8, (0, 255, 0), 1,
+                    cv2.LINE_AA)
+        cv2.putText(img_rd, "Q: Quit", (20, 450), self.font, 0.8, (255, 255, 255), 1, cv2.LINE_AA)
+
+        for i in range(len(self.current_frame_face_name_list)):
+            cv2.putText(img_rd, "Face " + str(i + 1), tuple(
+                [int(self.current_frame_centroid_list[i][0]), int(self.current_frame_centroid_list[i][1])]), self.font,
+                        0.8, (255, 190, 0),
+                        1,
+                        cv2.LINE_AA)
+
+    # 处理获取的视频流，进行人脸识别 / Face detection and recognition wit OT from input video stream
+    def process(self, stream):
+        # 1. 读取存放所有人脸特征的 csv / Get faces known from "features.all.csv"
+        if self.get_face_database():
+            while stream.isOpened():
+                self.frame_cnt += 1
+                print(">>> Frame " + str(self.frame_cnt) + " starts")
+                flag, img_rd = stream.read()
+                kk = cv2.waitKey(1)
+
+                # 2. 检测人脸 / Detect faces for frame X
+                faces = detector(img_rd, 0)
+                if self.current_frame_face_name_list == ['Person_2', 'Person_2']:
+                    break
+
+                # Update cnt for faces in frames
+                self.last_frame_face_cnt = self.current_frame_face_cnt
+                self.current_frame_face_cnt = len(faces)
+                # Update the face name list in last frame
+                self.last_frame_face_name_list = self.current_frame_face_name_list[:]
+                # update frame centroid list
+                self.last_frame_centroid_list = self.current_frame_centroid_list
+                self.current_frame_centroid_list = []
+                print("   >>> current_frame_face_cnt: ", self.current_frame_face_cnt)
+
+                # 2.1. if cnt not changes
+                if self.current_frame_face_cnt == self.last_frame_face_cnt:
+                    print("   >>> scene 1: 当前帧和上一帧相比没有发生人脸数变化 / no faces cnt changes in this frame!!!")
+                    self.current_frame_face_position_list = []
+                    if self.current_frame_face_cnt != 0:
+                        # 2.1.1 Get ROI positions
+                        for k, d in enumerate(faces):
+                            self.current_frame_face_position_list.append(tuple(
+                                [faces[k].left(), int(faces[k].bottom() + (faces[k].bottom() - faces[k].top()) / 4)]))
+                            self.current_frame_centroid_list.append(
+                                [int(faces[k].left() + faces[k].right()) / 2,
+                                 int(faces[k].top() + faces[k].bottom()) / 2])
+
+                            # 计算矩形框大小 / Compute the size of rectangle box
+                            height = (d.bottom() - d.top())
+                            width = (d.right() - d.left())
+                            hh = int(height / 2)
+                            ww = int(width / 2)
+                            cv2.rectangle(img_rd,
+                                          tuple([d.left() - ww, d.top() - hh]),
+                                          tuple([d.right() + ww, d.bottom() + hh]),
+                                          (255, 255, 255), 2)
+
+                    # multi-faces in current frames, use centroid tracker to track
+                    if self.current_frame_face_cnt != 1:
+                        self.centroid_tracker()
+
+                    for i in range(self.current_frame_face_cnt):
+                        # 6.2 write names under ROI
+                        cv2.putText(img_rd, self.current_frame_face_name_list[i],
+                                    self.current_frame_face_position_list[i], self.font, 0.8, (0, 255, 255), 1,
+                                    cv2.LINE_AA)
+
+                # 2.2 if cnt of faces changes, 0->1 or 1->0 or ...
+                else:
+                    print("   >>> scene 2: 当前帧和上一帧相比人脸数发生变化 / Faces cnt changes in this frame")
+                    self.current_frame_face_position_list = []
+                    self.current_frame_face_X_e_distance_list = []
+
+                    # 2.2.1 face cnt decrease: 1->0, 2->1, ...
+                    if self.current_frame_face_cnt == 0:
+                        print("   >>> scene 2.1 人脸消失, 当前帧中没有人脸 / No guy in this frame!!!")
+                        # clear list of names and features
+                        self.current_frame_face_name_list = []
+                        self.current_frame_face_features_list = []
+
+                    # 2.2.2 face cnt increase: 0->1, 0->2, ..., 1->2, ...
+                    else:
+                        print("   >>> scene 2.2 出现人脸，进行人脸识别 / Do face recognition for people detected in this frame")
+                        self.current_frame_face_name_list = []
+                        for i in range(len(faces)):
+                            shape = predictor(img_rd, faces[i])
+                            self.current_frame_face_features_list.append(
+                                face_reco_model.compute_face_descriptor(img_rd, shape))
+                            self.current_frame_face_name_list.append("unknown")
+
+                        # 2.2.2.1 遍历捕获到的图像中所有的人脸 / Traversal all the faces in the database
+                        for k in range(len(faces)):
+                            print("      >>> For face " + str(k+1) + " in current frame:")
+                            self.current_frame_centroid_list.append(
+                                [int(faces[k].left() + faces[k].right()) / 2,
+                                 int(faces[k].top() + faces[k].bottom()) / 2])
+
+                            self.current_frame_face_X_e_distance_list = []
+
+                            # 2.2.2.2 每个捕获人脸的名字坐标 / Positions of faces captured
+                            self.current_frame_face_position_list.append(tuple(
+                                [faces[k].left(), int(faces[k].bottom() + (faces[k].bottom() - faces[k].top()) / 4)]))
+
+                            # 2.2.2.3 对于某张人脸，遍历所有存储的人脸特征
+                            # For every faces detected, compare the faces in the database
+                            for i in range(len(self.features_known_list)):
+                                # 如果 person_X 数据不为空
+                                if str(self.features_known_list[i][0]) != '0.0':
+                                    print("            >>> with person", str(i + 1), "the e distance: ", end='')
+                                    e_distance_tmp = self.return_euclidean_distance(
+                                        self.current_frame_face_features_list[k],
+                                        self.features_known_list[i])
+                                    print(e_distance_tmp)
+                                    self.current_frame_face_X_e_distance_list.append(e_distance_tmp)
+                                else:
+                                    # 空数据 person_X
+                                    self.current_frame_face_X_e_distance_list.append(999999999)
+
+                            # 2.2.2.4 寻找出最小的欧式距离匹配 / Find the one with minimum e distance
+                            similar_person_num = self.current_frame_face_X_e_distance_list.index(
+                                min(self.current_frame_face_X_e_distance_list))
+
+                            if min(self.current_frame_face_X_e_distance_list) < 0.4:
+                                self.current_frame_face_name_list[k] = self.name_known_list[similar_person_num]
+                                print("            >>> recognition result for face " + str(k+1) +": "+ self.name_known_list[similar_person_num])
+                            else:
+                                print("            >>> recognition result for face " + str(k + 1) + ": " + "unknown")
+                # 3. 生成的窗口添加说明文字 / Add note on cv2 window
+                self.draw_note(img_rd)
+
+                # 4. 按下 'q' 键退出 / Press 'q' to exit
+                if kk == ord('q'):
+                    break
+
+                self.update_fps()
+                cv2.namedWindow("camera", 1)
+                cv2.imshow("camera", img_rd)
+                print(">>> Frame ends\n\n")
+
+    def run(self):
+        cap = cv2.VideoCapture(0)
+        self.process(cap)
+
+        cap.release()
+        cv2.destroyAllWindows()
+
+
+def main():
+    Face_Recognizer_con = Face_Recognizer()
+    Face_Recognizer_con.run()
+
+
+if __name__ == '__main__':
+    main()

face_reco_from_camera_ot_single_person.py

@@ -0,0 +1,247 @@
+# Copyright (C) 2020 coneypo
+# SPDX-License-Identifier: MIT
+
+# Author:   coneypo
+# Blog:     http://www.cnblogs.com/AdaminXie
+# GitHub:   https://github.com/coneypo/Dlib_face_recognition_from_camera
+# Mail:     coneypo@foxmail.com
+
+# 利用 OT 对于单张人脸追踪, 实时人脸识别 / Real-time face detection and recognition via OT for single face
+
+import dlib
+import numpy as np
+import cv2
+import os
+import pandas as pd
+import time
+
+# Dlib 正向人脸检测器 / Use frontal face detector of Dlib
+detector = dlib.get_frontal_face_detector()
+
+# Dlib 人脸 landmark 特征点检测器 / Get face landmarks
+predictor = dlib.shape_predictor('data/data_dlib/shape_predictor_68_face_landmarks.dat')
+
+# Dlib Resnet 人脸识别模型，提取 128D 的特征矢量 / Use Dlib resnet50 model to get 128D face descriptor
+face_reco_model = dlib.face_recognition_model_v1("data/data_dlib/dlib_face_recognition_resnet_model_v1.dat")
+
+
+class Face_Recognizer:
+    def __init__(self):
+        self.font = cv2.FONT_ITALIC
+
+        # For FPS
+        self.frame_time = 0
+        self.frame_start_time = 0
+        self.fps = 0
+
+        # cnt for frame
+        self.frame_cnt = 0
+
+        # 用来存储所有录入人脸特征的数组 / Save the features of faces in the database
+        self.features_known_list = []
+        # 用来存储录入人脸名字 / Save the name of faces in the database
+        self.name_known_list = []
+
+        # 用来存储上一帧和当前帧 ROI 的质心坐标 / List to save centroid positions of ROI in frame N-1 and N
+        self.last_frame_centroid_list = []
+        self.current_frame_centroid_list = []
+
+        # 用来存储上一帧和当前帧检测出目标的名字 / List to save names of objects in frame N-1 and N
+        self.last_frame_names_list = []
+        self.current_frame_face_names_list = []
+
+        # 上一帧和当前帧中人脸数的计数器 / cnt for faces in frame N-1 and N
+        self.last_frame_faces_cnt = 0
+        self.current_frame_face_cnt = 0
+
+        # 用来存放进行识别时候对比的欧氏距离 / Save the e-distance for faceX when recognizing
+        self.current_frame_face_X_e_distance_list = []
+
+        # 存储当前摄像头中捕获到的所有人脸的坐标名字 / Save the positions and names of current faces captured
+        self.current_frame_face_position_list = []
+        # 存储当前摄像头中捕获到的人脸特征 / Save the features of people in current frame
+        self.current_frame_face_features_list = []
+
+    # 从 "features_all.csv" 读取录入人脸特征 / Get known faces from "features_all.csv"
+    def get_face_database(self):
+        if os.path.exists("data/features_all.csv"):
+            path_features_known_csv = "data/features_all.csv"
+            csv_rd = pd.read_csv(path_features_known_csv, header=None)
+            for i in range(csv_rd.shape[0]):
+                features_someone_arr = []
+                for j in range(0, 128):
+                    if csv_rd.iloc[i][j] == '':
+                        features_someone_arr.append('0')
+                    else:
+                        features_someone_arr.append(csv_rd.iloc[i][j])
+                self.features_known_list.append(features_someone_arr)
+                self.name_known_list.append("Person_" + str(i + 1))
+            print("Faces in Database：", len(self.features_known_list))
+            return 1
+        else:
+            print('##### Warning #####', '\n')
+            print("'features_all.csv' not found!")
+            print(
+                "Please run 'get_faces_from_camera.py' and 'features_extraction_to_csv.py' before 'face_reco_from_camera.py'",
+                '\n')
+            print('##### End Warning #####')
+            return 0
+
+        # 计算两个128D向量间的欧式距离 / Compute the e-distance between two 128D features
+
+    # 更新 FPS / Update FPS of Video stream
+    def update_fps(self):
+        now = time.time()
+        self.frame_time = now - self.frame_start_time
+        self.fps = 1.0 / self.frame_time
+        self.frame_start_time = now
+        # 计算两个128D向量间的欧式距离 / Compute the e-distance between two 128D features
+
+    @staticmethod
+    def return_euclidean_distance(feature_1, feature_2):
+        feature_1 = np.array(feature_1)
+        feature_2 = np.array(feature_2)
+        dist = np.sqrt(np.sum(np.square(feature_1 - feature_2)))
+        return dist
+
+    # 生成的 cv2 window 上面添加说明文字 / putText on cv2 window
+    def draw_note(self, img_rd):
+        # 添加说明 / Add some statements
+        cv2.putText(img_rd, "Face Recognizer with OT (one person)", (20, 40), self.font, 1, (255, 255, 255), 1, cv2.LINE_AA)
+        cv2.putText(img_rd, "FPS:   " + str(self.fps.__round__(2)), (20, 100), self.font, 0.8, (0, 255, 0), 1,
+                    cv2.LINE_AA)
+        cv2.putText(img_rd, "Q: Quit", (20, 450), self.font, 0.8, (255, 255, 255), 1, cv2.LINE_AA)
+
+    # 处理获取的视频流，进行人脸识别 / Face detection and recognition wit OT from input video stream
+    def process(self, stream):
+        # 1. 读取存放所有人脸特征的 csv / Get faces known from "features.all.csv"
+        if self.get_face_database():
+            while stream.isOpened():
+                self.frame_cnt += 1
+                print(">>> Frame " + str(self.frame_cnt) + " starts")
+                flag, img_rd = stream.read()
+                kk = cv2.waitKey(1)
+
+                # 2. 检测人脸 / Detect faces for frame X
+                faces = detector(img_rd, 0)
+
+                # Update cnt for faces in frames
+                self.last_frame_faces_cnt = self.current_frame_face_cnt
+                self.current_frame_face_cnt = len(faces)
+                print("   >>> current_frame_face_cnt: ", self.current_frame_face_cnt)
+
+                # 2.1 If cnt not changes, 1->1 or 0->0
+                if self.current_frame_face_cnt == self.last_frame_faces_cnt:
+                    print("   >>> scene 1: 当前帧和上一帧相比没有发生人脸数变化 / no faces cnt changes in this frame!!!")
+                    # One face in this frame
+                    if self.current_frame_face_cnt != 0:
+                        # 2.1.1 Get ROI positions
+                        for k, d in enumerate(faces):
+                            # 计算矩形框大小 / Compute the size of rectangle box
+                            height = (d.bottom() - d.top())
+                            width = (d.right() - d.left())
+                            hh = int(height / 2)
+                            ww = int(width / 2)
+
+                            cv2.rectangle(img_rd,
+                                          tuple([d.left() - ww, d.top() - hh]),
+                                          tuple([d.right() + ww, d.bottom() + hh]),
+                                          (255, 255, 255), 2)
+
+                            self.current_frame_face_position_list[k] = tuple(
+                                [faces[k].left(), int(faces[k].bottom() + (faces[k].bottom() - faces[k].top()) / 4)])
+
+                            print("   >>> self.current_frame_face_names_list[k]:        ",
+                                  self.current_frame_face_names_list[k])
+                            print("   >>> self.current_frame_face_position_list[k]:     ",
+                                  self.current_frame_face_position_list[k])
+
+                            # 2.1.2 写名字 / Write names under ROI
+                            cv2.putText(img_rd, self.current_frame_face_names_list[k],
+                                        self.current_frame_face_position_list[k], self.font, 0.8, (0, 255, 255), 1,
+                                        cv2.LINE_AA)
+
+                # 2.2 if cnt of faces changes, 0->1 or 1->0
+                else:
+                    print("   >>> scene 2: 当前帧和上一帧相比人脸数发生变化 / Faces cnt changes in this frame")
+                    self.current_frame_face_position_list = []
+                    self.current_frame_face_X_e_distance_list = []
+
+                    # 2.2.1 face cnt: 1->0, no faces in this frame
+                    if self.current_frame_face_cnt == 0:
+                        print("   >>> scene 2.1 人脸消失, 当前帧中没有人脸 / no guy in this frame!!!")
+                        # clear list of names and
+                        self.current_frame_face_names_list = []
+                        self.current_frame_face_features_list = []
+
+                    # 2.2.2 face cnt: 0->1, get the new face
+                    elif self.current_frame_face_cnt == 1:
+                        print("   >>> scene 2.2 出现人脸，进行人脸识别 / Get person in this frame and do face recognition")
+                        self.current_frame_face_names_list = []
+
+                        for i in range(len(faces)):
+                            shape = predictor(img_rd, faces[i])
+                            self.current_frame_face_features_list.append(
+                                face_reco_model.compute_face_descriptor(img_rd, shape))
+
+                        # 2.2.2.1 遍历捕获到的图像中所有的人脸 / Traversal all the faces in the database
+                        for k in range(len(faces)):
+                            self.current_frame_face_names_list.append("unknown")
+
+                            # 2.2.2.2 每个捕获人脸的名字坐标 / Positions of faces captured
+                            self.current_frame_face_position_list.append(tuple(
+                                [faces[k].left(), int(faces[k].bottom() + (faces[k].bottom() - faces[k].top()) / 4)]))
+
+                            # 2.2.2.3 对于某张人脸，遍历所有存储的人脸特征
+                            # For every faces detected, compare the faces in the database
+                            for i in range(len(self.features_known_list)):
+                                # 如果 person_X 数据不为空
+                                if str(self.features_known_list[i][0]) != '0.0':
+                                    print("            >>> with person", str(i + 1), "the e distance: ", end='')
+                                    e_distance_tmp = self.return_euclidean_distance(
+                                        self.current_frame_face_features_list[k],
+                                        self.features_known_list[i])
+                                    print(e_distance_tmp)
+                                    self.current_frame_face_X_e_distance_list.append(e_distance_tmp)
+                                else:
+                                    # 空数据 person_X
+                                    self.current_frame_face_X_e_distance_list.append(999999999)
+
+                            # 2.2.2.4 寻找出最小的欧式距离匹配 / Find the one with minimum e distance
+                            similar_person_num = self.current_frame_face_X_e_distance_list.index(min(self.current_frame_face_X_e_distance_list))
+
+                            if min(self.current_frame_face_X_e_distance_list) < 0.4:
+                                self.current_frame_face_names_list[k] = self.name_known_list[similar_person_num]
+                                print("            >>> recognition result for face " + str(k + 1) + ": " +
+                                      self.name_known_list[similar_person_num])
+                            else:
+                                print("            >>> recognition result for face " + str(k + 1) + ": " + "unknown")
+
+                # 3. 生成的窗口添加说明文字 / Add note on cv2 window
+                self.draw_note(img_rd)
+
+                if kk == ord('q'):
+                    break
+
+                self.update_fps()
+
+                cv2.namedWindow("camera", 1)
+                cv2.imshow("camera", img_rd)
+
+                print(">>> Frame ends\n\n")
+
+    def run(self):
+        cap = cv2.VideoCapture(0)
+        self.process(cap)
+
+        cap.release()
+        cv2.destroyAllWindows()
+
+
+def main():
+    Face_Recognizer_con = Face_Recognizer()
+    Face_Recognizer_con.run()
+
+
+if __name__ == '__main__':
+    main()

features_extraction_to_csv.py

@@ -1,13 +1,12 @@
-# 从人脸图像文件中提取人脸特征存入 CSV
-# Features extraction from images and save into features_all.csv
+# Copyright (C) 2020 coneypo
+# SPDX-License-Identifier: MIT
 
 # Author:   coneypo
 # Blog:     http://www.cnblogs.com/AdaminXie
 # GitHub:   https://github.com/coneypo/Dlib_face_recognition_from_camera
 # Mail:     coneypo@foxmail.com
 
-# Created at 2018-05-11
-# Updated at 2020-04-02
+# 从人脸图像文件中提取人脸特征存入 "features_all.csv" / Extract features from images and save into "features_all.csv"
 
 import os
 import dlib
@@ -15,49 +14,51 @@ from skimage import io
 import csv
 import numpy as np
 
-# 要读取人脸图像文件的路径
+# 要读取人脸图像文件的路径 / Path of cropped faces
 path_images_from_camera = "data/data_faces_from_camera/"
 
-# 1. Dlib 正向人脸检测器
+# Dlib 正向人脸检测器 / Use frontal face detector of Dlib
 detector = dlib.get_frontal_face_detector()
 
-# 2. Dlib 人脸 landmark 特征点检测器
+# Dlib 人脸 landmark 特征点检测器 / Get face landmarks
 predictor = dlib.shape_predictor('data/data_dlib/shape_predictor_68_face_landmarks.dat')
 
-# 3. Dlib Resnet 人脸识别模型，提取 128D 的特征矢量
+# Dlib Resnet 人脸识别模型，提取 128D 的特征矢量 / Use Dlib resnet50 model to get 128D face descriptor
 face_reco_model = dlib.face_recognition_model_v1("data/data_dlib/dlib_face_recognition_resnet_model_v1.dat")
 
 
-# 返回单张图像的 128D 特征
+# 返回单张图像的 128D 特征 / Return 128D features for single image
+# Input:    path_img           <class 'str'>
+# Output:   face_descriptor    <class 'dlib.vector'>
 def return_128d_features(path_img):
     img_rd = io.imread(path_img)
     faces = detector(img_rd, 1)
 
     print("%-40s %-20s" % ("检测到人脸的图像 / Image with faces detected:", path_img), '\n')
 
-    # 因为有可能截下来的人脸再去检测，检测不出来人脸了
-    # 所以要确保是 检测到人脸的人脸图像 拿去算特征
+    # 因为有可能截下来的人脸再去检测，检测不出来人脸了, 所以要确保是 检测到人脸的人脸图像拿去算特征
+    # For photos of faces saved, we need to make sure that we can detect faces from the cropped images
     if len(faces) != 0:
         shape = predictor(img_rd, faces[0])
         face_descriptor = face_reco_model.compute_face_descriptor(img_rd, shape)
     else:
         face_descriptor = 0
         print("no face")
-
     return face_descriptor
 
 
-# 将文件夹中照片特征提取出来, 写入 CSV
+# 返回 personX 的 128D 特征均值 / Return the mean value of 128D face descriptor for person X
+# Input:    path_faces_personX       <class 'str'>
+# Output:   features_mean_personX    <class 'numpy.ndarray'>
 def return_features_mean_personX(path_faces_personX):
     features_list_personX = []
     photos_list = os.listdir(path_faces_personX)
     if photos_list:
         for i in range(len(photos_list)):
-            # 调用return_128d_features()得到128d特征
-            print("%-40s %-20s" % ("正在读的人脸图像 / Image to read:", path_faces_personX + "/" + photos_list[i]))
+            # 调用 return_128d_features() 得到 128D 特征 / Get 128D features for single image of personX
+            print("%-40s %-20s" % ("正在读的人脸图像 / Reading image:", path_faces_personX + "/" + photos_list[i]))
             features_128d = return_128d_features(path_faces_personX + "/" + photos_list[i])
-            #  print(features_128d)
-            # 遇到没有检测出人脸的图片跳过
+            # 遇到没有检测出人脸的图片跳过 / Jump if no face detected from image
             if features_128d == 0:
                 i += 1
             else:
@@ -65,17 +66,17 @@ def return_features_mean_personX(path_faces_personX):
     else:
         print("文件夹内图像文件为空 / Warning: No images in " + path_faces_personX + '/', '\n')
 
-    # 计算 128D 特征的均值
+    # 计算 128D 特征的均值 / Compute the mean
     # personX 的 N 张图像 x 128D -> 1 x 128D
     if features_list_personX:
         features_mean_personX = np.array(features_list_personX).mean(axis=0)
     else:
         features_mean_personX = np.zeros(128, dtype=int, order='C')
-
+    print(type(features_mean_personX))
     return features_mean_personX
 
 
-# 获取已录入的最后一个人脸序号 / get the num of latest person
+# 获取已录入的最后一个人脸序号 / Get the order of latest person
 person_list = os.listdir("data/data_faces_from_camera/")
 person_num_list = []
 for person in person_list:
@@ -91,4 +92,4 @@ with open("data/features_all.csv", "w", newline="") as csvfile:
         writer.writerow(features_mean_personX)
         print("特征均值 / The mean of features:", list(features_mean_personX))
         print('\n')
-    print("所有录入人脸数据存入 / Save all the features of faces registered into: data/features_all.csv")
+    print("所有录入人脸数据存入 / Save all the features of faces registered into: data/features_all.csv")

get_faces_from_camera.py

@@ -1,22 +1,21 @@
-# 进行人脸录入 / face register
-# 录入多张人脸 / support multi-faces
+# Copyright (C) 2020 coneypo
+# SPDX-License-Identifier: MIT
 
 # Author:   coneypo
 # Blog:     http://www.cnblogs.com/AdaminXie
 # GitHub:   https://github.com/coneypo/Dlib_face_recognition_from_camera
 # Mail:     coneypo@foxmail.com
 
-# Created at 2018-05-11
-# Updated at 2020-04-19
+# 进行人脸录入 / Face register
 
-import dlib         # 人脸处理的库 Dlib
-import numpy as np  # 数据处理的库 Numpy
-import cv2          # 图像处理的库 OpenCV
-import os           # 读写文件
-import shutil       # 读写文件
+import dlib
+import numpy as np
+import cv2
+import os
+import shutil
 import time
 
-# Dlib 正向人脸检测器
+# Dlib 正向人脸检测器 / Use frontal face detector of Dlib
 detector = dlib.get_frontal_face_detector()
 
 
@@ -25,22 +24,21 @@ class Face_Register:
         self.path_photos_from_camera = "data/data_faces_from_camera/"
         self.font = cv2.FONT_ITALIC
 
-        self.existing_faces_cnt = 0     # 已录入的人脸计数器
-        self.ss_cnt = 0                 # 录入 personX 人脸时图片计数器
-        self.faces_cnt = 0              # 录入人脸计数器
+        self.existing_faces_cnt = 0         # 已录入的人脸计数器 / cnt for counting saved faces
+        self.ss_cnt = 0                     # 录入 personX 人脸时图片计数器 / cnt for screen shots
+        self.current_frame_faces_cnt = 0    # 录入人脸计数器 / cnt for counting faces in current frame
 
-        # 之后用来控制是否保存图像的 flag / The flag to control if save
-        self.save_flag = 1
-        # 之后用来检查是否先按 'n' 再按 's' / The flag to check if press 'n' before 's'
-        self.press_n_flag = 0
+        self.save_flag = 1                  # 之后用来控制是否保存图像的 flag / The flag to control if save
+        self.press_n_flag = 0               # 之后用来检查是否先按 'n' 再按 's' / The flag to check if press 'n' before 's'
 
+        # FPS
         self.frame_time = 0
         self.frame_start_time = 0
         self.fps = 0
 
-    # 新建保存人脸图像文件和数据CSV文件夹 / Mkdir for saving photos and csv
+    # 新建保存人脸图像文件和数据CSV文件夹 / Make dir for saving photos and csv
     def pre_work_mkdir(self):
-        # 新建文件夹 / make folders to save faces images and csv
+        # 新建文件夹 / Create folders to save faces images and csv
         if os.path.isdir(self.path_photos_from_camera):
             pass
         else:
@@ -55,47 +53,46 @@ class Face_Register:
         if os.path.isfile("data/features_all.csv"):
             os.remove("data/features_all.csv")
 
-    # 如果有之前录入的人脸, 在之前 person_x 的序号按照 person_x+1 开始录入 /
-    # If the old folders exists, start from person_x+1
+    # 如果有之前录入的人脸, 在之前 person_x 的序号按照 person_x+1 开始录入 / Start from person_x+1
     def check_existing_faces_cnt(self):
         if os.listdir("data/data_faces_from_camera/"):
-            # 获取已录入的最后一个人脸序号 / Get the num of latest person
+            # 获取已录入的最后一个人脸序号 / Get the order of latest person
             person_list = os.listdir("data/data_faces_from_camera/")
             person_num_list = []
             for person in person_list:
                 person_num_list.append(int(person.split('_')[-1]))
             self.existing_faces_cnt = max(person_num_list)
 
-        # 如果第一次存储或者没有之前录入的人脸, 按照 person_1 开始录入
-        # Start from person_1
+        # 如果第一次存储或者没有之前录入的人脸, 按照 person_1 开始录入 / Start from person_1
         else:
             self.existing_faces_cnt = 0
 
-    # 获取处理之后 stream 的帧数 / Get the fps of video stream
+    # 获取处理之后 stream 的帧数 / Update FPS of video stream
     def update_fps(self):
         now = time.time()
         self.frame_time = now - self.frame_start_time
         self.fps = 1.0 / self.frame_time
         self.frame_start_time = now
 
-    # 生成的 cv2 window 上面添加说明文字 / putText on cv2 window
+    # 生成的 cv2 window 上面添加说明文字 / PutText on cv2 window
     def draw_note(self, img_rd):
-        # 添加说明 / Add some statements
+        # 添加说明 / Add some notes
         cv2.putText(img_rd, "Face Register", (20, 40), self.font, 1, (255, 255, 255), 1, cv2.LINE_AA)
         cv2.putText(img_rd, "FPS:   " + str(self.fps.__round__(2)), (20, 100), self.font, 0.8, (0, 255, 0), 1,
                     cv2.LINE_AA)
-        cv2.putText(img_rd, "Faces: " + str(self.faces_cnt), (20, 140), self.font, 0.8, (0, 255, 0), 1, cv2.LINE_AA)
+        cv2.putText(img_rd, "Faces: " + str(self.current_frame_faces_cnt), (20, 140), self.font, 0.8, (0, 255, 0), 1, cv2.LINE_AA)
         cv2.putText(img_rd, "N: Create face folder", (20, 350), self.font, 0.8, (255, 255, 255), 1, cv2.LINE_AA)
         cv2.putText(img_rd, "S: Save current face", (20, 400), self.font, 0.8, (255, 255, 255), 1, cv2.LINE_AA)
         cv2.putText(img_rd, "Q: Quit", (20, 450), self.font, 0.8, (255, 255, 255), 1, cv2.LINE_AA)
 
-    # 获取人脸
+    # 获取人脸 / Main process of face detection and saving
     def process(self, stream):
-        # 1. 新建储存人脸图像文件目录 / Uncomment if you need mkdir
-        # self.pre_work_mkdir()
+        # 1. 新建储存人脸图像文件目录 / Create folders to save photos
+        self.pre_work_mkdir()
 
-        # 2. 删除 "/data/data_faces_from_camera" 中已有人脸图像文件 / Uncomment if want to delete the old faces
-        self.pre_work_del_old_face_folders()
+        # 2. 删除 "/data/data_faces_from_camera" 中已有人脸图像文件 / Uncomment if want to delete the saved faces and start from person_1
+        if os.path.isdir(self.path_photos_from_camera):
+            self.pre_work_del_old_face_folders()
 
         # 3. 检查 "/data/data_faces_from_camera" 中已有人脸文件
         self.check_existing_faces_cnt()
@@ -103,7 +100,7 @@ class Face_Register:
         while stream.isOpened():
             flag, img_rd = stream.read()        # Get camera video stream
             kk = cv2.waitKey(1)
-            faces = detector(img_rd, 0)         # Use dlib face detector
+            faces = detector(img_rd, 0)         # Use Dlib face detector
 
             # 4. 按下 'n' 新建存储人脸的文件夹 / Press 'n' to create the folders for saving faces
             if kk == ord('n'):
@@ -113,12 +110,12 @@ class Face_Register:
                 print('\n')
                 print("新建的人脸文件夹 / Create folders: ", current_face_dir)
 
-                self.ss_cnt = 0              # 将人脸计数器清零 / clear the cnt of faces
-                self.press_n_flag = 1        # 已经按下 'n' / have pressed 'n'
+                self.ss_cnt = 0                 # 将人脸计数器清零 / Clear the cnt of screen shots
+                self.press_n_flag = 1           # 已经按下 'n' / Pressed 'n' already
 
             # 5. 检测到人脸 / Face detected
             if len(faces) != 0:
-                # 矩形框 / Show the HOG of faces
+                # 矩形框 / Show the ROI of faces
                 for k, d in enumerate(faces):
                     # 计算矩形框大小 / Compute the size of rectangle box
                     height = (d.bottom() - d.top())
@@ -126,7 +123,7 @@ class Face_Register:
                     hh = int(height/2)
                     ww = int(width/2)
 
-                    # 6. 判断人脸矩形框是否超出 480x640
+                    # 6. 判断人脸矩形框是否超出 480x640 / If the size of ROI > 480x640
                     if (d.right()+ww) > 640 or (d.bottom()+hh > 480) or (d.left()-ww < 0) or (d.top()-hh < 0):
                         cv2.putText(img_rd, "OUT OF RANGE", (20, 300), self.font, 0.8, (0, 0, 255), 1, cv2.LINE_AA)
                         color_rectangle = (0, 0, 255)
@@ -142,7 +139,7 @@ class Face_Register:
                                   tuple([d.right() + ww, d.bottom() + hh]),
                                   color_rectangle, 2)
 
-                    # 7. 根据人脸大小生成空的图像 / Create blank image according to the shape of face detected
+                    # 7. 根据人脸大小生成空的图像 / Create blank image according to the size of face detected
                     img_blank = np.zeros((int(height*2), width*2, 3), np.uint8)
 
                     if save_flag:
@@ -158,7 +155,8 @@ class Face_Register:
                                 print("写入本地 / Save into：", str(current_face_dir) + "/img_face_" + str(self.ss_cnt) + ".jpg")
                             else:
                                 print("请先按 'N' 来建文件夹, 按 'S' / Please press 'N' and press 'S'")
-                self.faces_cnt = len(faces)
+
+            self.current_frame_faces_cnt = len(faces)
 
             # 9. 生成的窗口添加说明文字 / Add note on cv2 window
             self.draw_note(img_rd)
@@ -167,7 +165,9 @@ class Face_Register:
             if kk == ord('q'):
                 break
 
+            # 11. Update FPS
             self.update_fps()
+
             cv2.namedWindow("camera", 1)
             cv2.imshow("camera", img_rd)
 

how_to_use_camera.py

@@ -1,6 +1,3 @@
-# OpenCV 调用摄像头
-# 默认调用笔记本摄像头
-
 # Author:   coneypo
 # Blog:     http://www.cnblogs.com/AdaminXie
 # GitHub:   https://github.com/coneypo/Dlib_face_recognition_from_camera
@@ -34,7 +31,7 @@ cap = cv2.VideoCapture(0)
 18. cv2.CAP_PROP_RECTIFICATION Rectification flag for stereo cameras (note: only supported by DC1394 v 2.x backend currently)
 """
 
-# The default shape of camera will be 640x480 in Windows or Ubuntu
+# The default size of frame from camera will be 640x480 in Windows or Ubuntu
 # So we will not set "cap.set" here, it doesn't work
 # cap.set(propId=cv2.CAP_PROP_FRAME_WIDTH, value=cap.get(cv2.CAP_PROP_FRAME_WIDTH))
 
@@ -84,4 +81,4 @@ while cap.isOpened():
 cap.release()
 
 # 删除建立的所有窗口
-cv2.destroyAllWindows()
+cv2.destroyAllWindows()

requirements.txt

@@ -1,3 +1,3 @@
-dlib==19.17.0
+31231dlib==19.17.0
 numpy==1.15.1
 scikit-image==0.14.0