【英语学习系统】使用kinect v2进行动作捕捉
系统中在学生学习完单词后会有一个巩固单词做题的小测试，测试为选择题的方式，因为系统是投影到用户面前的大屏幕上的所以选择使用kinect对用户动作进行捕捉，判断用户的选择。
选择题界面：
kinect开发：
系统中使用kinect v2进行动作捕捉，首先需要下载kinect v2 sdk
官方下载地址：https://www.microsoft.com/en-us/download/details.aspx?id=44561
下载安装后得到：
打开SDK Browser
因为整个系统是采用c#语言进行开发的，选用Sample:c#，因为我们的目的是通过获取用户动作判断用户的选择，所以选择
Body Basic-WPF
选择install，得到一个Windows的标准wpf程序，文件结构如下：
使用Visual Studio打开.sln文件：
这个脚本中包含了对骨骼点的记录，通过建立数组对判断用户动作所需的骨骼点数据进行保存：
用于判断用户输入的方法：
//通过七个骨骼节点在K帧内的位置判断所做动作对应的ABC选项
public int PointWhere(point[] HandTipLeft, point[] HandTipRight,point[] Spine,point[] ShoulderCenter, point[] ShoulderLeft, point[] ShoulderRight,point[] HipCenter,int i, int k)
{
int A = 0;
int B = 0;
int C = 0;
for(; k>0&&i-k>0; k--)
{
double midY = (ShoulderCenter[i-k].y + HipCenter[i-k].y) / 2.0;
if (HandTipRight[i-k].y <= midY && HandTipLeft[i - k].y <= midY) continue;
double X = 0;
if (HandTipLeft[i - k].y > midY) X = HandTipLeft[i - k].x;
if (HandTipRight[i - k].y > midY) X = HandTipRight[i - k].x;
if (X != 0)
{
if (X < ShoulderLeft[i - k].x) A++;
if (X > ShoulderLeft[i - k].x && HandTipRight[i - k].x < ShoulderRight[i - k].x) B++;
if (X > ShoulderRight[i - k].x) C++;
}
}
if (A <= 30 && B <= 30 && C <= 30) return 0;
if (A > B && A > C) return 1;
if (B > A && B > C) return 2;
return 3;
}通过得到k帧的数据将用户的右手与驱赶夹角判断用户手臂的指向以判断用户的输入。
在unity下的系统项目中的小测试场景的脚本与该程序进行数据通信，使unity项目可以实时使用该程序的判断结果。
动作判断的完整脚本如下
//------------------------------------------------------------------------------
//
// Copyright (c) Microsoft Corporation. All rights reserved.
//
//------------------------------------------------------------------------------
namespace Microsoft.Samples.Kinect.BodyBasics
{
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Windows;
using System.Windows.Media;
using System.Windows.Media.Imaging;
using Microsoft.Kinect;
using System.Threading;
///
/// Interaction logic for MainWindow
///
public partial class MainWindow : Window, INotifyPropertyChanged
{
int i = 0;
point[] HandTipLeft = new point[200000];
point[] HandTipRight = new point[200000];
point[] Spine = new point[200000];
point[] ShoulderCenter = new point[200000];
point[] ShoulderLeft = new point[200000];
point[] ShoulderRight = new point[200000];
point[] HipCenter = new point[200000];
///
/// Radius of drawn hand circles
///
private const double HandSize = 30;
///
/// Thickness of drawn joint lines
///
private const double JointThickness = 3;
///
/// Thickness of clip edge rectangles
///
private const double ClipBoundsThickness = 10;
///
/// Constant for clamping Z values of camera space points from being negative
///
private const float InferredZPositionClamp = 0.1f;
///
/// Brush used for drawing hands that are currently tracked as closed
///
private readonly Brush handClosedBrush = new SolidColorBrush(Color.FromArgb(128, 255, 0, 0));
///
/// Brush used for drawing hands that are currently tracked as opened
///
private readonly Brush handOpenBrush = new SolidColorBrush(Color.FromArgb(128, 0, 255, 0));
///
/// Brush used for drawing hands that are currently tracked as in lasso (pointer) position
///
private readonly Brush handLassoBrush = new SolidColorBrush(Color.FromArgb(128, 0, 0, 255));
///
/// Brush used for drawing joints that are currently tracked
///
private readonly Brush trackedJointBrush = new SolidColorBrush(Color.FromArgb(255, 68, 192, 68));
///
/// Brush used for drawing joints that are currently inferred
///
private readonly Brush inferredJointBrush = Brushes.Yellow;
///
/// Pen used for drawing bones that are currently inferred
///
private readonly Pen inferredBonePen = new Pen(Brushes.Gray, 1);
///
/// Drawing group for body rendering output
///
private DrawingGroup drawingGroup;
///
/// Drawing image that we will display
///
private DrawingImage imageSource;
///
/// Active Kinect sensor
///
private KinectSensor kinectSensor = null;
///
/// Coordinate mapper to map one type of point to another
///
private CoordinateMapper coordinateMapper = null;
///
/// Reader for body frames
///
private BodyFrameReader bodyFrameReader = null;
///
/// Array for the bodies
///
private Body[] bodies = null;
///
/// definition of bones
///
private List> bones;
///
/// Width of display (depth space)
///
private int displayWidth;
///
/// Height of display (depth space)
///
private int displayHeight;
///
/// List of colors for each body tracked
///
private List bodyColors;
///
/// Current status text to display
///
private string statusText = null;
///
/// Initializes a new instance of the MainWindow class.
///
///
public struct point
{
public double x, y, z;
}
public MainWindow()
{
Thread t = new Thread(new ThreadStart(DoChoose));
t.Start();
// one sensor is currently supported
this.kinectSensor = KinectSensor.GetDefault();
// get the coordinate mapper
this.coordinateMapper = this.kinectSensor.CoordinateMapper;
// get the depth (display) extents
FrameDescription frameDescription = this.kinectSensor.DepthFrameSource.FrameDescription;
// get size of joint space
this.displayWidth = frameDescription.Width;
this.displayHeight = frameDescription.Height;
// open the reader for the body frames
this.bodyFrameReader = this.kinectSensor.BodyFrameSource.OpenReader();
// a bone defined as a line between two joints
this.bones = new List>();
// Torso
this.bones.Add(new Tuple(JointType.Head, JointType.Neck));
this.bones.Add(new Tuple(JointType.Neck, JointType.SpineShoulder));
this.bones.Add(new Tuple(JointType.SpineShoulder, JointType.SpineMid));
this.bones.Add(new Tuple(JointType.SpineMid, JointType.SpineBase));
this.bones.Add(new Tuple(JointType.SpineShoulder, JointType.ShoulderRight));
this.bones.Add(new Tuple(JointType.SpineShoulder, JointType.ShoulderLeft));
this.bones.Add(new Tuple(JointType.SpineBase, JointType.HipRight));
this.bones.Add(new Tuple(JointType.SpineBase, JointType.HipLeft));
// Right Arm
this.bones.Add(new Tuple(JointType.ShoulderRight, JointType.ElbowRight));
this.bones.Add(new Tuple(JointType.ElbowRight, JointType.WristRight));
this.bones.Add(new Tuple(JointType.WristRight, JointType.HandRight));
this.bones.Add(new Tuple(JointType.HandRight, JointType.HandTipRight));
this.bones.Add(new Tuple(JointType.WristRight, JointType.ThumbRight));
// Left Arm
this.bones.Add(new Tuple(JointType.ShoulderLeft, JointType.ElbowLeft));
this.bones.Add(new Tuple(JointType.ElbowLeft, JointType.WristLeft));
this.bones.Add(new Tuple(JointType.WristLeft, JointType.HandLeft));
this.bones.Add(new Tuple(JointType.HandLeft, JointType.HandTipLeft));
this.bones.Add(new Tuple(JointType.WristLeft, JointType.ThumbLeft));
// Right Leg
this.bones.Add(new Tuple(JointType.HipRight, JointType.KneeRight));
this.bones.Add(new Tuple(JointType.KneeRight, JointType.AnkleRight));
this.bones.Add(new Tuple(JointType.AnkleRight, JointType.FootRight));
// Left Leg
this.bones.Add(new Tuple(JointType.HipLeft, JointType.KneeLeft));
this.bones.Add(new Tuple(JointType.KneeLeft, JointType.AnkleLeft));
this.bones.Add(new Tuple(JointType.AnkleLeft, JointType.FootLeft));
// populate body colors, one for each BodyIndex
this.bodyColors = new List();
this.bodyColors.Add(new Pen(Brushes.Red, 6));
this.bodyColors.Add(new Pen(Brushes.Orange, 6));
this.bodyColors.Add(new Pen(Brushes.Green, 6));
this.bodyColors.Add(new Pen(Brushes.Blue, 6));
this.bodyColors.Add(new Pen(Brushes.Indigo, 6));
this.bodyColors.Add(new Pen(Brushes.Violet, 6));
// set IsAvailableChanged event notifier
this.kinectSensor.IsAvailableChanged += this.Sensor_IsAvailableChanged;
// open the sensor
this.kinectSensor.Open();
// set the status text
this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
: Properties.Resources.NoSensorStatusText;
// Create the drawing group we'll use for drawing
this.drawingGroup = new DrawingGroup();
// Create an image source that we can use in our image control
this.imageSource = new DrawingImage(this.drawingGroup);
// use the window object as the view model in this simple example
this.DataContext = this;
// initialize the components (controls) of the window
this.InitializeComponent();
}
///
/// INotifyPropertyChangedPropertyChanged event to allow window controls to bind to changeable data
///
public event PropertyChangedEventHandler PropertyChanged;
///
/// Gets the bitmap to display
///
public ImageSource ImageSource
{
get
{
return this.imageSource;
}
}
///
/// Gets or sets the current status text to display
///
public string StatusText
{
get
{
return this.statusText;
}
set
{
if (this.statusText != value)
{
this.statusText = value;
// notify any bound elements that the text has changed
if (this.PropertyChanged != null)
{
this.PropertyChanged(this, new PropertyChangedEventArgs("StatusText"));
}
}
}
}
///
/// Execute start up tasks
///
/// object sending the event
/// event arguments
private void MainWindow_Loaded(object sender, RoutedEventArgs e)
{
if (this.bodyFrameReader != null)
{
this.bodyFrameReader.FrameArrived += this.Reader_FrameArrived;
}
}
///
/// Execute shutdown tasks
///
/// object sending the event
/// event arguments
private void MainWindow_Closing(object sender, CancelEventArgs e)
{
if (this.bodyFrameReader != null)
{
// BodyFrameReader is IDisposable
this.bodyFrameReader.Dispose();
this.bodyFrameReader = null;
}
if (this.kinectSensor != null)
{
this.kinectSensor.Close();
this.kinectSensor = null;
}
}
///
/// Handles the body frame data arriving from the sensor
///
/// object sending the event
/// event arguments
private void Reader_FrameArrived(object sender, BodyFrameArrivedEventArgs e)
{
bool dataReceived = false;
using (BodyFrame bodyFrame = e.FrameReference.AcquireFrame())
{
if (bodyFrame != null)
{
if (this.bodies == null)
{
this.bodies = new Body[bodyFrame.BodyCount];
}
// The first time GetAndRefreshBodyData is called, Kinect will allocate each Body in the array.
// As long as those body objects are not disposed and not set to null in the array,
// those body objects will be re-used.
bodyFrame.GetAndRefreshBodyData(this.bodies);
dataReceived = true;
}
}
if (dataReceived)
{
using (DrawingContext dc = this.drawingGroup.Open())
{
// Draw a transparent background to set the render size
dc.DrawRectangle(Brushes.Black, null, new Rect(0.0, 0.0, this.displayWidth, this.displayHeight));
int penIndex = 0;
foreach (Body body in this.bodies)
{
Pen drawPen = this.bodyColors[penIndex++];
if (body.IsTracked)
{
this.DrawClippedEdges(body, dc);
IReadOnlyDictionary joints = body.Joints;
// convert the joint points to depth (display) space
Dictionary jointPoints = new Dictionary();
HandTipLeft[i].x = body.Joints[JointType.HandTipLeft ].Position.X;
HandTipLeft[i].y = body.Joints[JointType.HandTipLeft].Position.Y;
HandTipRight[i].x = body.Joints[JointType.HandTipRight].Position.X;
HandTipRight[i].y = body.Joints[JointType.HandTipRight].Position.Y;
Spine[i].x = body.Joints[JointType.SpineMid].Position.X;
Spine[i].y = body.Joints[JointType.SpineMid].Position.Y;
ShoulderCenter[i].x = body.Joints[JointType.SpineShoulder].Position.X;
ShoulderCenter[i].y = body.Joints[JointType.SpineShoulder].Position.Y;
ShoulderLeft[i].x = body.Joints[JointType.ShoulderLeft].Position.X;
ShoulderLeft[i].y = body.Joints[JointType.ShoulderLeft].Position.Y;
ShoulderRight[i].x = body.Joints[JointType.ShoulderRight].Position.X;
ShoulderRight[i].y = body.Joints[JointType.ShoulderRight].Position.Y;
HipCenter[i].x = body.Joints[JointType.SpineBase].Position.X;
HipCenter[i].y = body.Joints[JointType.SpineBase].Position.Y;
i++;
foreach (JointType jointType in joints.Keys)
{
// sometimes the depth(Z) of an inferred joint may show as negative
// clamp down to 0.1f to prevent coordinatemapper from returning (-Infinity, -Infinity)
CameraSpacePoint position = joints[jointType].Position;
if (position.Z < 0)
{
position.Z = InferredZPositionClamp;
}
DepthSpacePoint depthSpacePoint = this.coordinateMapper.MapCameraPointToDepthSpace(position);
jointPoints[jointType] = new Point(depthSpacePoint.X, depthSpacePoint.Y);
}
this.DrawBody(joints, jointPoints, dc, drawPen);
this.DrawHand(body.HandLeftState, jointPoints[JointType.HandLeft], dc);
this.DrawHand(body.HandRightState, jointPoints[JointType.HandRight], dc);
}
}
// prevent drawing outside of our render area
this.drawingGroup.ClipGeometry = new RectangleGeometry(new Rect(0.0, 0.0, this.displayWidth, this.displayHeight));
}
}
}
///
/// Draws a body
///
/// joints to draw
/// translated positions of joints to draw
/// drawing context to draw to
/// specifies color to draw a specific body
private void DrawBody(IReadOnlyDictionary joints, IDictionary jointPoints, DrawingContext drawingContext, Pen drawingPen)
{
// Draw the bones
foreach (var bone in this.bones)
{
this.DrawBone(joints, jointPoints, bone.Item1, bone.Item2, drawingContext, drawingPen);
}
// Draw the joints
foreach (JointType jointType in joints.Keys)
{
Brush drawBrush = null;
TrackingState trackingState = joints[jointType].TrackingState;
if (trackingState == TrackingState.Tracked)
{
drawBrush = this.trackedJointBrush;
}
else if (trackingState == TrackingState.Inferred)
{
drawBrush = this.inferredJointBrush;
}
if (drawBrush != null)
{
drawingContext.DrawEllipse(drawBrush, null, jointPoints[jointType], JointThickness, JointThickness);
}
}
}
///
/// Draws one bone of a body (joint to joint)
///
/// joints to draw
/// translated positions of joints to draw
/// first joint of bone to draw
/// second joint of bone to draw
/// drawing context to draw to
/// /// specifies color to draw a specific bone
private void DrawBone(IReadOnlyDictionary joints, IDictionary jointPoints, JointType jointType0, JointType jointType1, DrawingContext drawingContext, Pen drawingPen)
{
Joint joint0 = joints[jointType0];
Joint joint1 = joints[jointType1];
// If we can't find either of these joints, exit
if (joint0.TrackingState == TrackingState.NotTracked ||
joint1.TrackingState == TrackingState.NotTracked)
{
return;
}
// We assume all drawn bones are inferred unless BOTH joints are tracked
Pen drawPen = this.inferredBonePen;
if ((joint0.TrackingState == TrackingState.Tracked) && (joint1.TrackingState == TrackingState.Tracked))
{
drawPen = drawingPen;
}
drawingContext.DrawLine(drawPen, jointPoints[jointType0], jointPoints[jointType1]);
}
///
/// Draws a hand symbol if the hand is tracked: red circle = closed, green circle = opened; blue circle = lasso
///
/// state of the hand
/// position of the hand
/// drawing context to draw to
private void DrawHand(HandState handState, Point handPosition, DrawingContext drawingContext)
{
switch (handState)
{
case HandState.Closed:
drawingContext.DrawEllipse(this.handClosedBrush, null, handPosition, HandSize, HandSize);
break;
case HandState.Open:
drawingContext.DrawEllipse(this.handOpenBrush, null, handPosition, HandSize, HandSize);
break;
case HandState.Lasso:
drawingContext.DrawEllipse(this.handLassoBrush, null, handPosition, HandSize, HandSize);
break;
}
}
///
/// Draws indicators to show which edges are clipping body data
///
/// body to draw clipping information for
/// drawing context to draw to
private void DrawClippedEdges(Body body, DrawingContext drawingContext)
{
FrameEdges clippedEdges = body.ClippedEdges;
if (clippedEdges.HasFlag(FrameEdges.Bottom))
{
drawingContext.DrawRectangle(
Brushes.Red,
null,
new Rect(0, this.displayHeight - ClipBoundsThickness, this.displayWidth, ClipBoundsThickness));
}
if (clippedEdges.HasFlag(FrameEdges.Top))
{
drawingContext.DrawRectangle(
Brushes.Red,
null,
new Rect(0, 0, this.displayWidth, ClipBoundsThickness));
}
if (clippedEdges.HasFlag(FrameEdges.Left))
{
drawingContext.DrawRectangle(
Brushes.Red,
null,
new Rect(0, 0, ClipBoundsThickness, this.displayHeight));
}
if (clippedEdges.HasFlag(FrameEdges.Right))
{
drawingContext.DrawRectangle(
Brushes.Red,
null,
new Rect(this.displayWidth - ClipBoundsThickness, 0, ClipBoundsThickness, this.displayHeight));
}
}
///
/// Handles the event which the sensor becomes unavailable (E.g. paused, closed, unplugged).
///
/// object sending the event
/// event arguments
private void Sensor_IsAvailableChanged(object sender, IsAvailableChangedEventArgs e)
{
// on failure, set the status text
this.StatusText = this.kinectSensor.IsAvailable ? Properties.Resources.RunningStatusText
: Properties.Resources.SensorNotAvailableStatusText;
}
//通过七个骨骼节点在K帧内的位置判断所做动作对应的ABC选项
public int PointWhere(point[] HandTipLeft, point[] HandTipRight,point[] Spine,point[] ShoulderCenter, point[] ShoulderLeft, point[] ShoulderRight,point[] HipCenter,int i, int k)
{
int A = 0;
int B = 0;
int C = 0;
for(; k>0&&i-k>0; k--)
{
double midY = (ShoulderCenter[i-k].y + HipCenter[i-k].y) / 2.0;
if (HandTipRight[i-k].y <= midY && HandTipLeft[i - k].y <= midY) continue;
double X = 0;
if (HandTipLeft[i - k].y > midY) X = HandTipLeft[i - k].x;
if (HandTipRight[i - k].y > midY) X = HandTipRight[i - k].x;
if (X != 0)
{
if (X < ShoulderLeft[i - k].x) A++;
if (X > ShoulderLeft[i - k].x && HandTipRight[i - k].x < ShoulderRight[i - k].x) B++;
if (X > ShoulderRight[i - k].x) C++;
}
}
if (A <= 30 && B <= 30 && C <= 30) return 0;
if (A > B && A > C) return 1;
if (B > A && B > C) return 2;
return 3;
}
public void DoChoose()
{
Console.WriteLine("开始");
int outABC = 0;
while (true)
{
StreamReader sr = new StreamReader(@".\question.txt");
Thread.Sleep(500);
if (sr.ReadLine().IndexOf( "123")==-1)//当测试开始时进入判断ABC方法
{
continue;
}
Thread.Sleep(1500);//先等待1.5秒等用户做动作
StreamWriter sw = new StreamWriter(@".\abc.txt");
outABC = PointWhere(HandTipLeft, HandTipRight, Spine, ShoulderCenter, ShoulderLeft, ShoulderRight, HipCenter, i, 60);
while (outABC == 0)//如果未得到有效结果则再次调用
{
Thread.Sleep(500);
outABC = PointWhere(HandTipLeft, HandTipRight, Spine, ShoulderCenter, ShoulderLeft, ShoulderRight, HipCenter, i, 60);
}
sw.WriteLine(outABC);//输出选择结果
Console.WriteLine("输出"+ outABC);//测试
sw.Close();
}
}
}
}