Kinect for windows的脸部识别-白红宇

Kinect for windows的脸部识别

阅读量：6239 次

发布时间：2019-06-22

本文共 9441 字，大约阅读时间需要 31 分钟。

Kinect for windows提供了脸部识别的功能，可以识出人脸。主要是通过FaceTrackFrame类的GetTriangles()来得到一个三角形数组，这个三角形数组就是给成人面部的基本形状，并且组成的效果是立体的（可以这样理解，可以把3D都拆成三角形来表示，看上去像3D，但其实是2D），这个数组的每个元素都存放着三个整数，分别代码三角形的第一个点，第二个点和第三个点。FaceTrackFrame的GetProjected3DShape方法，可以获取一组坐标信息，这样就可以结合三角形数组元素中的点作为索引，从本方法的坐标集合中取出每个三角形的坐标点来了，就可以绘制这些三角形，就可以组成一个人脸的网络3D效果图了。

本例是从色彩摄像头中获取彩色数据流，并显示到窗体上，再通过FaceTracker得到得到脸部3D信息，并用GDI+的方式把网络图形画到窗体上，这时就可以在真实图像上看到浮着一张网络的面套。同时可以得出脸部交汇最多的坐标，并用GDI+添加上不同有色彩，同时还可以得到人体面部和Kinect 正面的偏差，即人头是否竖直，有没有偏上一边等角度信息。

结果图像如下图：

新建一个WinForm项目，在窗体上添加一人ListBox，更改Name为Messbox_LB。后台代码如下：

using System;

using System.Collections.Generic;

using System.ComponentModel;

using System.Data;

using System.Drawing;

using System.Linq;

using System.Text;

using System.Threading.Tasks;

using System.Windows.Forms;

using Microsoft.Kinect;

using Microsoft.Kinect.Toolkit;

using Microsoft.Kinect.Toolkit.FaceTracking;

using System.Threading;

using System.IO;

using System.Drawing.Imaging;

namespace Face

{

public partial class Form1 : Form

{

public Form1()

{

InitializeComponent();

}

KinectSensor ks = null;

private void Form1_Load(object sender, EventArgs e)

{

//让winform窗体刷新不闪动

this.SetStyle(ControlStyles.OptimizedDoubleBuffer, true);

this.SetStyle(ControlStyles.AllPaintingInWmPaint, true);

this.SetStyle(ControlStyles.UserPaint, true);

this.SetStyle(ControlStyles.DoubleBuffer, true);

//找到连接的Kinect设备

foreach (var ks in KinectSensor.KinectSensors)

{

if (ks.Status == KinectStatus.Connected)

{

this.ks = ks;

}

}

//开启色彩流，深度流，骨骼流的跟踪

if (this.ks != null)

{

this.ks.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);

this.ks.DepthStream.Enable(DepthImageFormat.Resolution320x240Fps30);

this.ks.DepthStream.Range = DepthRange.Near;

this.ks.SkeletonStream.EnableTrackingInNearRange = true;

this.ks.SkeletonStream.TrackingMode = SkeletonTrackingMode.Seated;

this.ks.SkeletonStream.Enable();

//订阅跟踪数据读取事件

this.ks.AllFramesReady += OnAllFramesReady;

ks.Start();

}

}

//这个方法很重要，就是重绘人脸跟踪采集到的数据

protected override void OnPaint(PaintEventArgs e)

{

base.OnPaint(e);

foreach (SkeletonFaceTracker faceInformation in this.trackedSkeletons.Values)

{

//第一个参数为当前窗体为画布，第二个是添加采集到的信息到listbox中，这个方法画识别到脸部的信息

faceInformation.DrawFaceModel(e.Graphics, Messbox_LB);

}

}

//定义脸部识别的集合

private readonly Dictionary<int, SkeletonFaceTracker> trackedSkeletons = new Dictionary<int, SkeletonFaceTracker>();

//色彩流字节数组

private byte[] colorImage;

private ColorImageFormat colorImageFormat = ColorImageFormat.Undefined;

//深度流字节数组

private short[] depthImage;

private DepthImageFormat depthImageFormat = DepthImageFormat.Undefined;

//骨骼信息数组

private Skeleton[] skeletonData;

private void OnAllFramesReady(object sender, AllFramesReadyEventArgs allFramesReadyEventArgs)

{

ColorImageFrame colorImageFrame = null;

DepthImageFrame depthImageFrame = null;

SkeletonFrame skeletonFrame = null;

{

colorImageFrame = allFramesReadyEventArgs.OpenColorImageFrame(); //接到色彩流对框架

depthImageFrame = allFramesReadyEventArgs.OpenDepthImageFrame(); //接到深度流对框架

skeletonFrame = allFramesReadyEventArgs.OpenSkeletonFrame(); //接到骨骼流对框架

if (colorImageFrame == null || depthImageFrame == null || skeletonFrame == null)

{

return;

}

if (this.depthImageFormat != depthImageFrame.Format)

{

this.ResetFaceTracking();

this.depthImage = null;

this.depthImageFormat = depthImageFrame.Format;

}

if (this.colorImageFormat != colorImageFrame.Format)

{

this.ResetFaceTracking();

this.colorImage = null;

this.colorImageFormat = colorImageFrame.Format;

}

if (this.depthImage == null)

{

this.depthImage = new short[depthImageFrame.PixelDataLength];

}

if (this.colorImage == null)

{

this.colorImage = new byte[colorImageFrame.PixelDataLength];

}

if (this.skeletonData == null || this.skeletonData.Length != skeletonFrame.SkeletonArrayLength)

{

this.skeletonData = new Skeleton[skeletonFrame.SkeletonArrayLength];

}

//获取各种数据流信息

colorImageFrame.CopyPixelDataTo(this.colorImage);

depthImageFrame.CopyPixelDataTo(this.depthImage);

skeletonFrame.CopySkeletonDataTo(this.skeletonData);

//清空列表信息

Messbox_LB.Items.Clear();

//编历骨骼流

foreach (Skeleton skeleton in this.skeletonData)

{

//找到有效的骨骼信息

if (skeleton.TrackingState == SkeletonTrackingState.Tracked

|| skeleton.TrackingState == SkeletonTrackingState.PositionOnly)

{

if (!this.trackedSkeletons.ContainsKey(skeleton.TrackingId))

{

//添加骨骼信息到集合中

this.trackedSkeletons.Add(skeleton.TrackingId, new SkeletonFaceTracker());

}

// 得到脸部识别对象

SkeletonFaceTracker skeletonFaceTracker;

if (this.trackedSkeletons.TryGetValue(skeleton.TrackingId, out skeletonFaceTracker))

{

//把获取的数据流的相关信息传给OnFrameReady方法

skeletonFaceTracker.OnFrameReady(Messbox_LB, this.ks, colorImageFormat, colorImage, depthImageFormat, depthImage, skeleton);

skeletonFaceTracker.LastTrackedFrame = skeletonFrame.FrameNumber;

}

}

}

//这个刷新会触发窗体的重画，OnPaint方法会被调用。

this.Refresh();

//把色彩流转转成位图显示成窗体的背景

this.BackgroundImage = ToGrayBitmap(colorImage, 640, 480);

}

finally

{

if (colorImageFrame != null)

{

colorImageFrame.Dispose();

}

if (depthImageFrame != null)

{

depthImageFrame.Dispose();

}

if (skeletonFrame != null)

{

skeletonFrame.Dispose();

}

}

}

//把色采流数据转成位图返回

public static Bitmap ToGrayBitmap(byte[] rawValues, int width, int height)

{

//定议转换图片的格式，一个像素占32个，前24位为红绿蓝，后8位为空

PixelFormat pf = PixelFormat.Format32bppRgb;

//申请目标位图的变量

Bitmap bmp = new Bitmap(width, height, pf);

//将其内存区域锁定

BitmapData bmpData = bmp.LockBits(new Rectangle(0, 0, width, height), ImageLockMode.WriteOnly, pf);

//获取位图的起始地址

IntPtr iptr = bmpData.Scan0;

//用Marshal的Copy方法，将色彩流字节数组复制到BitmapData中

System.Runtime.InteropServices.Marshal.Copy(rawValues, 0, iptr, rawValues.Length);

//释放锁

bmp.UnlockBits(bmpData);

return bmp;

}

//重新设置识别对象

private void ResetFaceTracking()

{

foreach (int trackingId in new List<int>(this.trackedSkeletons.Keys))

{

this.RemoveTracker(trackingId);

}

}

//从集合中移动识别信息

private void RemoveTracker(int trackingId)

{

this.trackedSkeletons[trackingId].Dispose();

this.trackedSkeletons.Remove(trackingId);

}

private void Form1_FormClosing(object sender, FormClosingEventArgs e)

{

if (this.ks.Status == KinectStatus.Connected)

{

ks.Stop();

}

}

//定义脸识别类

class SkeletonFaceTracker : IDisposable

{

//定义脸部识别形状三角形数组

private static FaceTriangle[] faceTriangles;

//脸部识别坐标点集合

private EnumIndexableCollection<FeaturePoint, Microsoft.Kinect.Toolkit.FaceTracking.PointF> facePoints;

//脸部跟踪类

private FaceTracker faceTracker;

//定义识别成功标识

private bool lastFaceTrackSucceeded;

//骨骼跟踪状态

private SkeletonTrackingState skeletonTrackingState;

public int LastTrackedFrame { get; set; }

public void Dispose()

{

if (this.faceTracker != null)

{

this.faceTracker.Dispose();

this.faceTracker = null;

}

}

//用来把识别的信息绘制出来

public void DrawFaceModel(Graphics graphics, ListBox lb)

{

if (!this.lastFaceTrackSucceeded || this.skeletonTrackingState != SkeletonTrackingState.Tracked)

{

return;

}

List<System.Drawing.PointF> faceModelPts = new List<System.Drawing.PointF>();

for (int i = 0; i < this.facePoints.Count; i++)

{

faceModelPts.Add(new System.Drawing.PointF(this.facePoints[i].X + 0.5f, this.facePoints[i].Y + 0.5f));

}

System.Drawing.Pen pen = new System.Drawing.Pen(System.Drawing.Color.Green);

List<System.Drawing.PointF> list = new List<System.Drawing.PointF>();

//遍历所有的三角形，分别画三角形

for (int i = 0; i < faceTriangles.Count(); i++)

{

System.Drawing.PointF[] pointFarr = new System.Drawing.PointF[4];

pointFarr[0] = faceModelPts[faceTriangles[i].First];

pointFarr[1] = faceModelPts[faceTriangles[i].Second];

pointFarr[2] = faceModelPts[faceTriangles[i].Third];

pointFarr[3] = faceModelPts[faceTriangles[i].First];

list.AddRange(pointFarr.Take(3));

graphics.DrawLines(pen, pointFarr);

}

lb.Items.Add(list.GroupBy(f => f).Count() + "点");

int count = list.GroupBy(f => f).Max(s => s.Count());

lb.Items.Add(count);

foreach (var v in list.GroupBy(f => f).Where(s => s.Count() == 10))

{

lb.Items.Add(v.Key + " " + 10);

graphics.FillEllipse(new SolidBrush(System.Drawing.Color.Red), v.Key.X, v.Key.Y, 5, 5);

}

foreach (var v in list.GroupBy(f => f).Where(s => s.Count() == 9))

{

lb.Items.Add(v.Key + " " + 9);

graphics.FillEllipse(new SolidBrush(System.Drawing.Color.Blue), v.Key.X, v.Key.Y, 5, 5);

}

foreach (var v in list.GroupBy(f => f).Where(s => s.Count() == 8))

{

lb.Items.Add(v.Key + " " + 8);

graphics.FillEllipse(new SolidBrush(System.Drawing.Color.Black), v.Key.X, v.Key.Y, 5, 5);

}

}

/// <summary>

/// 数据更新的方法

/// </summary>

internal void OnFrameReady(ListBox lb, KinectSensor kinectSensor, ColorImageFormat colorImageFormat, byte[] colorImage, DepthImageFormat depthImageFormat, short[] depthImage, Skeleton skeletonOfInterest)