face alignment 5个关键点对齐

访问网址查看 人脸对齐的发展

https://max.book118.com/html/2017/1111/139651668.shtm

1.准备数据集：UMD 把 21个关键点 提取出5个。

2.Loss 函数使用 absloss ，如何添加 caffe absloss 查看我的其他博文。

C:\WorkSpace\SoftWare\caffe\convert_multilabel_imageset.exe --shuffle D:\UMD\result_img_96\ D:\UMD\result_img_96\new_shutter_96_5_train.txt E:\work\face_alignment\lanmark_5\model\umd_96_5\lmdb2\train_data_npd E:\work\face_alignment\lanmark_5\model\umd_96_5\lmdb2\train_label_npd 15

C:\WorkSpace\SoftWare\caffe\convert_multilabel_imageset.exe D:\UMD\result_img_96\ D:\UMD\result_img_96\shutter_96_5_val.txt E:\work\face_alignment\lanmark_5\model\umd_96_5\lmdb2\test_data_npd E:\work\face_alignment\lanmark_5\model\umd_96_5\lmdb2\test_label_npd 15

pause

# The train/test net protocol buffer definition
net: "train_5_op_abs.prototxt"
# test_iter specifies how many forward passes the test should carry out.
# In the case of MNIST, we have test batch size 100 and 100 test iterations,
# covering the full 10,000 testing images.
test_iter: 48
# Carry out testing every 500 training iterations.
test_interval: 1000
# The base learning rate, momentum and the weight decay of the network.
base_lr: 0.0004
#base_lr:0.0002
#加快梯度下降的速度
momentum: 0.95
#momentum2: 0.999
weight_decay: 0.0001
# The learning rate policy
lr_policy: "inv"
gamma: 0.0001
power: 0.75
# Display every 100 iterations
display: 1000
# The maximum number of iterations
max_iter: 300000
# snapshot intermediate results
snapshot: 10000
snapshot_prefix: "save_path_abs/"
# solver mode: CPU or GPU
solver_mode: GPU

#type:"Adam" 

name: "Faceregressions"
layer {
  name: "landmark_5"
  type: "Data"
  top: "data"
  include {
    phase: TRAIN
  }
  transform_param {
		mean_value: 104
		mean_value: 117
		mean_value: 124
		scale: 0.0078125
  }
  data_param {
    source: "lmdb/train_data_npd"
    batch_size: 64
    backend: LMDB
  }
}
layer {
  name: "landmark_5"
  type: "Data"
  top: "label"
  include {
    phase: TRAIN
  }
  data_param {
    source: "lmdb/train_label_npd"
    batch_size: 64
    backend: LMDB
  }
}

layer {
  name: "landmark_5"
  type: "Data"
  top: "data"
  include {
    phase: TEST
  }
  #transform_param {
   # scale: 0.00390625
  #}
  transform_param {
		mean_value: 104
		mean_value: 117
		mean_value: 124
		scale: 0.0078125
  }
  data_param {
    source: "lmdb/test_data_npd"
    batch_size: 64
    backend: LMDB
  }
}
layer {
  name: "landmark_5"
  type: "Data"
  top: "label"
  include {
    phase: TEST
  }
  data_param {
    source: "lmdb/test_label_npd"
    batch_size: 64
    backend: LMDB
  }
}
layer {
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 32
    kernel_size: 3
    pad:1
    stride: 2
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu1"
  type: "ReLU"
  bottom: "conv1"
  top: "conv1"
}

layer {
  name: "conv2"
  type: "Convolution"
  bottom: "conv1"
  top: "conv2"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 64
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu2"
  type: "ReLU"
  bottom: "conv2"
  top: "conv2"
}

layer {
  name: "conv3"
  type: "Convolution"
  bottom: "conv2"
  top: "conv3"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 128
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu3"
  type: "ReLU"
  bottom: "conv3"
  top: "conv3"
}

layer {
  name: "conv4"
  type: "Convolution"
  bottom: "conv3"
  top: "conv4"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 128
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu4"
  type: "ReLU"
  bottom: "conv4"
  top: "conv4"
}

layer {
  name: "conv5"
  type: "Convolution"
  bottom: "conv4"
  top: "conv5"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 128
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu5"
  type: "ReLU"
  bottom: "conv5"
  top: "conv5"
}
layer {
  name: "conv6"
  type: "Convolution"
  bottom: "conv5"
  top: "conv6"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 128
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu6"
  type: "ReLU"
  bottom: "conv6"
  top: "conv6"
}

layer {
  name: "ip1"
  type: "InnerProduct"
  bottom: "conv6"
  top: "ip1"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  inner_product_param {
    num_output: 256
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu7"
  type: "ReLU"
  bottom: "ip1"
  top: "ip1"
}
layer {
  name: "ip2"
  type: "InnerProduct"
  bottom: "ip1"
  top: "ip2"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  inner_product_param {
    num_output: 10
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}

layer {
  name: "slice"
  type: "Slice"
  bottom: "label"
  top: "label1"
  top: "label2"
  slice_param {
    axis: 1
    slice_point: 10
  }
}
# 5个 visible 信息暂时没用 label2

layer {
  name: "accuracy"
  type: "AbsoluteLoss"
  bottom: "ip2"
  bottom: "label1"
  top: "accuracy"
  include {
    phase: TEST
  }
}

layer {
  name: "loss"
  type: "AbsoluteLoss"
  bottom: "ip2"
  bottom: "label1"
  top: "loss"
}

name: "Faceregressions"
layer
{  
  name: "data"  
  type: "MemoryData"
  top: "data"
  top: "label"
  memory_data_param   
  {
    batch_size: 1
    channels: 3
    height: 96
    width: 96
  }
  transform_param {
		mean_value: 104
		mean_value: 117
		mean_value: 124
		scale: 0.0078125
  }
}
layer {
  name: "conv1"
  type: "Convolution"
  bottom: "data"
  top: "conv1"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 32
    kernel_size: 3
    pad:1
    stride: 2
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu1"
  type: "ReLU"
  bottom: "conv1"
  top: "conv1"
}

layer {
  name: "conv2"
  type: "Convolution"
  bottom: "conv1"
  top: "conv2"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 64
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu2"
  type: "ReLU"
  bottom: "conv2"
  top: "conv2"
}

layer {
  name: "conv3"
  type: "Convolution"
  bottom: "conv2"
  top: "conv3"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 128
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu3"
  type: "ReLU"
  bottom: "conv3"
  top: "conv3"
}

layer {
  name: "conv4"
  type: "Convolution"
  bottom: "conv3"
  top: "conv4"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 128
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu4"
  type: "ReLU"
  bottom: "conv4"
  top: "conv4"
}

layer {
  name: "conv5"
  type: "Convolution"
  bottom: "conv4"
  top: "conv5"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 128
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu5"
  type: "ReLU"
  bottom: "conv5"
  top: "conv5"
}
layer {
  name: "conv6"
  type: "Convolution"
  bottom: "conv5"
  top: "conv6"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  convolution_param {
    num_output: 128
    kernel_size: 3
    stride: 2
    pad:1
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu6"
  type: "ReLU"
  bottom: "conv6"
  top: "conv6"
}

layer {
  name: "ip1"
  type: "InnerProduct"
  bottom: "conv6"
  top: "ip1"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  inner_product_param {
    num_output: 256
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}
layer {
  name: "relu7"
  type: "ReLU"
  bottom: "ip1"
  top: "ip1"
}
layer {
  name: "ip2"
  type: "InnerProduct"
  bottom: "ip1"
  top: "ip2"
  param {
    lr_mult: 1
  }
  param {
    lr_mult: 2
  }
  inner_product_param {
    num_output: 10
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "constant"
    }
  }
}

 

 

 

 

 

#include "Util.h"
#include 
#include 
#include 
#include   //使用stringstream需要引入这个头文件  
#include 
#include 
#include 
#include 
#include 
#include "../../../../SoftWare/libsvm-3.22/svm.h"

using namespace std;
using namespace glasssix;
using namespace cv;

const int LEFT_EYE = 0;
const int RIGHT_EYE = 9;
const int NOSE = 34;
const int LEFT_MOUTH = 46;
const int RIGHT_MOUTH = 47;
const int ALL = 95;
template 
Type stringToNum(const string& str)
{
	istringstream iss(str);
	Type num;
	iss >> num;
	return num;
}
void getFiles(string path, vector& files, vector& filenames, const string & tail)
{
	//文件句柄  
	intptr_t hFile = 0;
	//文件信息  
	struct _finddata_t fileinfo;
	string p;
	if ((hFile = _findfirst(p.assign(path).append("\\*").c_str(), &fileinfo)) != -1)
	{
		do
		{
			//如果是目录,迭代之  
			//如果不是,加入列表  
			if ((fileinfo.attrib &  _A_SUBDIR))
			{
				if (strcmp(fileinfo.name, ".") != 0 && strcmp(fileinfo.name, "..") != 0)
				{
					getFiles(p.assign(path).append("\\").append(fileinfo.name), files, filenames, tail);
				}

			}
			else
			{
				string filename = fileinfo.name;
				size_t found = filename.find(tail);
				if (found != string::npos)
				{
					files.push_back(p.assign(path).append("\\").append(filename));
					filenames.push_back(filename);
				}
			}
		} while (_findnext(hFile, &fileinfo) == 0);

		_findclose(hFile);
	}
}
void  splitString(const string& s, vector& v, const string& c)
{
	string::size_type pos1, pos2;
	pos2 = s.find(c);
	pos1 = 0;
	while (string::npos != pos2)
	{
		v.push_back(s.substr(pos1, pos2 - pos1));
		pos1 = pos2 + c.size();
		pos2 = s.find(c, pos1);
	}
	if (pos1 != s.length()) {
		v.push_back(s.substr(pos1));
	}
}
void parseData_5(string buf, LandMark& mark)
{
	vector result1;
	splitString(buf, result1, "\t");
	mark.fileName = result1[0];
	vector result2;
	splitString(result1[1], result2, ",");
	mark.points[0].x = stringToNum(result2[2 * LEFT_EYE]);
	mark.points[0].y = stringToNum(result2[2 * LEFT_EYE + 1]);
	mark.points[1].x = stringToNum(result2[2 * RIGHT_EYE]);
	mark.points[1].y = stringToNum(result2[2 * RIGHT_EYE + 1]);
	mark.points[2].x = stringToNum(result2[2 * NOSE]);
	mark.points[2].y = stringToNum(result2[2 * NOSE + 1]);
	mark.points[3].x = stringToNum(result2[2 * LEFT_MOUTH]);
	mark.points[3].y = stringToNum(result2[2 * LEFT_MOUTH + 1]);
	mark.points[4].x = stringToNum(result2[2 * RIGHT_MOUTH]);
	mark.points[4].y = stringToNum(result2[2 * RIGHT_MOUTH + 1]);

}
void parseKey_26_Data(string buf, LandMark& mark)
{
	vector result1;
	splitString(buf, result1, "\t");
	mark.fileName = result1[0];
	vector result2;
	splitString(result1[1], result2, ",");

	mark.points[0].x = stringToNum(result2[2 * LEFT_EYE]);
	mark.points[0].y = stringToNum(result2[2 * LEFT_EYE + 1]);
	mark.points[1].x = stringToNum(result2[2 * RIGHT_EYE]);
	mark.points[1].y = stringToNum(result2[2 * RIGHT_EYE + 1]);
	mark.points[2].x = stringToNum(result2[2 * NOSE]);
	mark.points[2].y = stringToNum(result2[2 * NOSE + 1]);
	mark.points[3].x = stringToNum(result2[2 * LEFT_MOUTH]);
	mark.points[3].y = stringToNum(result2[2 * LEFT_MOUTH + 1]);
	mark.points[4].x = stringToNum(result2[2 * RIGHT_MOUTH]);
	mark.points[4].y = stringToNum(result2[2 * RIGHT_MOUTH + 1]);
	int j = 5;
	for (int i = 74; i < ALL; i++)
	{
		if (i >= 74 && i < 95) {
			mark.points[j].x = stringToNum(result2[2 * i]);
			mark.points[j].y = stringToNum(result2[2 * i + 1]);
			j++;
		}
	}
}
void parseKey_68_Data(string buf, LandMark& mark)
{
	vector result1;
	splitString(buf, result1, "\t");
	mark.fileName = result1[0];
	vector result2;
	splitString(result1[1], result2, ",");

	int j = 0;
	//add left eye   18个
	for (int i = 0; i < 37; i++)
	{
		if ((i >= 0 && i < 9) || (i >= 18 && i < 26) || (i == 36)) {
			mark.points[j].x = stringToNum(result2[2 * i]);
			mark.points[j].y = stringToNum(result2[2 * i + 1]);
			j++;
		}
	}
	//add right eye  18个
	for (int i = 9; i < 38; i++)
	{
		if ((i >= 9 && i < 18) || (i >= 26 && i < 34) || (i == 37)) {
			mark.points[j].x = stringToNum(result2[2 * i]);
			mark.points[j].y = stringToNum(result2[2 * i + 1]);
			j++;
		}
	}
	//add nose eye  10个
	for (int i = 34; i < 46; i++)
	{
		if ((i >= 34 && i < 36) || (i >= 38 && i < 46)) {
			mark.points[j].x = stringToNum(result2[2 * i]);
			mark.points[j].y = stringToNum(result2[2 * i + 1]);
			j++;
		}
	}
	//add mouth eye  22个
	for (int i = 46; i < 68; i++)
	{
		if (i >= 46 && i < 68) {
			mark.points[j].x = stringToNum(result2[2 * i]);
			mark.points[j].y = stringToNum(result2[2 * i + 1]);
			j++;
		}
	}
}

void parseData(string buf, LandMark& mark, int num_mark)
{
	vector result1;
	splitString(buf, result1, " ");
	mark.fileName = result1[0];
	for (int i = 0; i < num_mark; i++)
	{
		mark.points[i].x = stringToNum(result1[2 * i + 1]);
		mark.points[i].y = stringToNum(result1[2 * (i + 1)]);
	}
}

void parseData(string buf, LandMark& mark)
{
	vector result1;
	splitString(buf, result1, "\t");
	mark.fileName = result1[0];
	vector result2;
	splitString(result1[1], result2, ",");
	for (int i = 0; i < LANDMARK_NUM; i++)
	{
		mark.points[i].x = stringToNum(result2[2 * i]);
		mark.points[i].y = stringToNum(result2[2 * i + 1]);
	}
}

void DataPrepareUtil::clearFileData(string filePath)
{
	if (filePath == "")
	{
		return;
	}
	ofstream in;
	in.open(filePath, ios::trunc);
	in.close();
}

void DataPrepareUtil::readallFile(std::string dir, vector & files, vector& filenames, string const  & tail)
{
	getFiles(dir, files, filenames, tail);
}
int DataPrepareUtil::getpts(string filename, LandMark & landmark)
{
	if (filename != "")
	{
		vector result;
		splitString(filename, result, ".");
		string file = result[0] + ".pts";
		ifstream fileA(file);
		if (!fileA)
		{
			cout << "没有找到需要读取的  " << file << " 请将文件放到指定位置再次运行本程序。" << endl;
			return -1;
		}
		//landmark.fileName = filename;
		for (int i = 0; !fileA.eof(); i++)
		{
			string buf;
			getline(fileA, buf, '\n');

			if (buf == "")
			{
				//cout << "buf is empty." << endl;
				continue;
			}
			if (i >= 3 && i < 71)
			{
				vector point;
				splitString(buf, point, " ");
				landmark.points[i - 3].x = stringToNum(point[0]);
				landmark.points[i - 3].y = stringToNum(point[1]);
			}
		}
		fileA.close();
		return 0;
	}
	else
	{
		cout << "file name is null." << endl;
		return -1;
	}
}

void DataPrepareUtil::readLabelData(string filePath, std::vector &data, int count)
{
	ifstream fileA(filePath);
	if (!fileA)
	{
		cout << "没有找到需要读取的  " << filePath << " 请将文件放到指定位置再次运行本程序。" << endl << "  按任意键以退出";
		return;
	}
	for (int i = 0; !fileA.eof() && (i < count); i++)
	{
		Label mark;
		string buf;
		getline(fileA, buf, '\n');

		if (buf == "")
		{
			cout << "buf is empty." << endl;
			continue;
		}
		vector resu;
		splitString(buf, resu, " ");
		mark.fileName = resu[0];
		mark.label = resu[1];
		data.push_back(mark);
	}
	fileA.close();
}
void parsePointVisibleData(string buf, LandMark& mark, int landmark)
{
	vector result1;
	splitString(buf, result1, " ");
	mark.fileName = result1[0];
	for (int i = 0; i < landmark * 2; i++)
	{
		if (i < landmark)
		{
			mark.points[i].x = stringToNum(result1[2 * i + 1]);
			mark.points[i].y = stringToNum(result1[2 * i + 2]);
		}
		else {
			mark.visiable[i - landmark] = stringToNum(result1[landmark + i + 1]);
		}
	}
}

vector DataPrepareUtil::readStageOneData(string filePath, int numMark, int count)
{
	vector result;
	ifstream fileA(filePath);
	if (!fileA)
	{
		cout << "没有找到需要读取的  " << filePath << " 请将文件放到指定位置再次运行本程序。" << endl << "  按任意键以退出";
		return result;
	}
	for (int i = 0; !fileA.eof() && (i < count); i++)
	{
		LandMark mark;
		string buf;
		getline(fileA, buf, '\n');

		if (buf == "")
		{
			cout << "buf is empty." << endl;
			continue;
		}
		parseData(buf, mark, numMark);

		result.push_back(mark);
	}
	fileA.close();

	return result;
}
vector DataPrepareUtil::readPointVisibleData(string filePath, int numMark, int count)
{
	vector result;
	ifstream fileA(filePath);
	if (!fileA)
	{
		cout << "没有找到需要读取的  " << filePath << " 请将文件放到指定位置再次运行本程序。" << endl << "  按任意键以退出";
		return result;
	}
	for (int i = 0; !fileA.eof() && (i < count); i++)
	{
		LandMark mark;
		string buf;
		getline(fileA, buf, '\n');

		if (buf == "")
		{
			cout << "buf is empty." << endl;
			continue;
		}
		parsePointVisibleData(buf, mark, numMark);
		result.push_back(mark);
	}
	fileA.close();
	return result;
}
void DataPrepareUtil::writeDatatoFile(std::string filePath, vector & data)
{
	if (filePath == "" || data.size() == 0)
	{
		return;
	}
	ofstream in;
	in.open(filePath, ios::app);   //ios::trunc
	int length = data.size();
	for (int i = 0; i < length; i++)
	{
		string dataline = "H:\\CASIA\\CASIA-WebFace\\" + data[i].fileName;
		dataline.append(" ");
		dataline.append(data[i].label);
		in << dataline << "\n";
	}
	in.close();
}
void DataPrepareUtil::writeDatatoFile(std::string filePath, vector & data, int landmark)
{
	if (filePath == "" || data.size() == 0)
	{
		return;
	}
	ofstream in;
	in.open(filePath, ios::app);
	int length = data.size();
	for (int i = 0; i < length; i++)
	{
		string dataline = data[i].fileName;
		dataline.append(" ");
		for (int j = 0; j < landmark; j++)
		{
			dataline.append(to_string(data[i].points[j].x));
			dataline.append(" ");
			dataline.append(to_string(data[i].points[j].y));
			if (j != (landmark - 1))
			{
				dataline.append(" ");
			}
			//cout.precision(20);
			//cout << "double x:" << data[i].points[j].x << endl;
			//cout.precision(20); // 设置输出精度
			//cout << "to_string():" << doubleToString(data[i].points[j].x)< & data, int landmark)
{
	if (filePath == "" || data.size() == 0)
	{
		return;
	}
	ofstream in;
	in.open(filePath, ios::app);
	int length = data.size();
	for (int i = 0; i < length; i++)
	{
		string dataline = data[i].fileName;
		dataline.append(" ");
		for (int j = 0; j < landmark; j++)
		{
			dataline.append(to_string(data[i].points[j].x));
			dataline.append(" ");
		}
		for (int j = 0; j < landmark; j++)
		{
			dataline.append(to_string(data[i].points[j].y));
			if (j != (landmark - 1))
			{
				dataline.append(" ");
			}
		}
		in << dataline << "\n";
	}
	in.close();
}


void DataPrepareUtil::writeDatatoFile(std::string filePath, vector & data, int landmark, int start, int end)
{
	if (filePath == "" || data.size() == 0)
	{
		return;
	}
	ofstream in;
	in.open(filePath, ios::app);
	int length = data.size();

	if (start < 0 || start > length || end < start || end > length)
	{
		cout << "start or end is error." << endl;
	}

	for (int i = start; i < end; i++)
	{
		string dataline = data[i].fileName;
		dataline.append(" ");
		for (int j = 0; j < landmark; j++)
		{
			dataline.append(to_string(data[i].points[j].x));
			dataline.append(" ");
			dataline.append(to_string(data[i].points[j].y));
			if (j != (landmark - 1))
			{
				dataline.append(" ");
			}
			//cout.precision(20);
			//cout << "double x:" << data[i].points[j].x << endl;
			//cout.precision(20); // 设置输出精度
			//cout << "to_string():" << doubleToString(data[i].points[j].x)< sift = xfeatures2d::SIFT::create(1);
	std::vector keypointsa;
	keypointsa.clear();

	KeyPoint keyp;
	keyp.pt.x = point.x;
	keyp.pt.y = point.y;
	keyp.size = 16;
	keypointsa.push_back(keyp);

	//sift->detectAndCompute(src, mask, keypointsa, a);//得到特征点和特征点描述
	//drawKeypoints(src, keypointsa, src, Scalar(0,0,255));//画出特征点
	sift->detectAndCompute(img, Mat(), keypointsa, desc, true);
	//drawKeypoints(img, keypointsa, img, Scalar(0, 0, 255));
	/*imshow("src", img);
	waitKey(0);*/
}

int DataPrepareUtil::getImageHogFeature(Mat &img, vector & descriptors)
{
	if (img.data == NULL)
	{
		cout << "No exist" << endl;
		return -1;
	}
	HOGDescriptor *hog = new HOGDescriptor(Size(8, 8), Size(8, 8), Size(8, 8), Size(4, 4), 9);   //Size(4,4) cell 大小
	hog->compute(img, descriptors, Size(1, 1), Size(0, 0));
	return 0;
}

void check_xy(int &x, int &y, int width, int height, int stride)
{
	if (x < 0)
	{
		x = 0;
	}
	if (y < 0)
	{
		y = 0;
	}
	if (x > width - stride)
	{
		x = width - stride;
	}
	if (y > height - stride)
	{
		y = height - stride;
	}
}
void DataPrepareUtil::PCA_Reduce(cv::Mat & input_train, cv::Mat & input_test, cv::Mat & output_train, cv::Mat & output_test, Mat & mean, Mat & eigenvectors)
{
	cout << "start pca" << endl;
	double t0 = (double)cvGetTickCount();
	PCA pca(input_train, Mat(), PCA::DATA_AS_ROW, 1700);
	cout << "end pca" << endl;
	double t1 = (double)cvGetTickCount();
	cout << "cost time is: " << ((t1 - t0) / ((double)cvGetTickFrequency() * 1000 * 1000)) << "s" << endl;
	//cout << pca.eigenvalues << endl;
	//cout << pca.eigenvectors << endl;
	output_train = pca.project(input_train);
	output_test = pca.project(input_test);
	cout << " point size :" << output_train.rows << " " << output_train.cols << endl;

	//imwrite("D:\\face\\face_img_96\\img\\feature_sift\\pca\\mean.jpg", pca.mean);
	//imwrite("D:\\face\\face_img_96\\img\\feature_sift\\pca\\engv.jpg", pca.eigenvectors);
	mean = pca.mean;
	eigenvectors = pca.eigenvectors;
}
void DataPrepareUtil::getData(std::string base, std::vector & really, vector & predict, cv::Mat & train_data, cv::Mat & train_label, cv::Mat & test_data, cv::Mat & test_label, int train_num, int test_num, int flage)
{
	int length = predict.size();
	cout << "train image is :" << length << endl;
	int n = 0;
	int featureNum = 0;
	int cols = 68 * 2;
	for (int i = 0; i < length; i++)
	{
		Mat img = imread(base + predict[i].fileName, CV_LOAD_IMAGE_COLOR);
		std::vector descriptors;
		if (flage == HOG_FEATURE)
		{
			for (int j = 0; j < 68; j++)
			{
				std::vector descriptor;

				Rect rect;
				rect.x = predict[i].points[j].x;
				rect.y = predict[i].points[j].y;
				rect.x = rect.x - 4;
				rect.y = rect.y - 4;
				check_xy(rect.x, rect.y, 96, 96, 8);
				rect.width = 8;
				rect.height = 8;
				Mat roi = img(rect);
				getImageHogFeature(roi, descriptor);
				int le = descriptor.size();
				for (int k = 0; k < le; k++)
				{
					descriptors.push_back(descriptor[k]);
				}
			}
		}
		else if (flage == SIFT_FEATURE)
		{
			for (int j = 0; j < LANDMARK_NUM; j++)
			{
				Mat desc;
				sift_feature(Point(predict[i].points[j].x, predict[i].points[j].y), desc, img);
				for (int k = 0; k < desc.cols; k++)
				{
					descriptors.push_back(desc.at(0, k));
				}
				//cout << " value :" << desc << endl;
				//system("PAUSE");
			}
		}
		if (i < train_num)
		{
			if (i == 0)
			{
				featureNum = descriptors.size();
				cout << "featureNum is:" << featureNum << endl;
				train_label = Mat::zeros(train_num, cols, CV_32FC1); //注意其中训练和自动训练的接口，还有labelMat一定要用CV_32SC1的类型
				train_data = Mat::zeros(train_num, descriptors.size(), CV_32FC1);
			}
			for (int j = 0; j < cols / 2; j++)
			{
				train_label.at(i, 2 * j) = really[i].points[j].x - predict[i].points[j].x;
				train_label.at(i, 2 * j + 1) = really[i].points[j].y - predict[i].points[j].y;
				//cout << train_label.at(i, 2 * j) << endl;
			}
			n = 0;
			for (std::vector::iterator iter = descriptors.begin(); iter != descriptors.end(); iter++)
			{
				train_data.at(i, n) = *iter;
				n++;
			}
		}
		else
		{
			if (i == train_num)
			{
				featureNum = descriptors.size();
				cout << "test featureNum is:" << featureNum << endl;
				test_label = Mat::zeros(test_num, cols, CV_32FC1);
				test_data = Mat::zeros(test_num, descriptors.size(), CV_32FC1);
			}
			for (int j = 0; j < cols / 2; j++)
			{
				test_label.at(i - train_num, 2 * j) = really[i - train_num].points[j].x - predict[i - train_num].points[j].x;
				test_label.at(i - train_num, 2 * j + 1) = really[i - train_num].points[j].y - predict[i - train_num].points[j].y;
			}
			n = 0;
			for (std::vector::iterator iter = descriptors.begin(); iter != descriptors.end(); iter++)
			{
				test_data.at(i - train_num, n) = *iter;
				n++;
			}
		}
	}
}

void DataPrepareUtil::getPatchData(std::string base, std::vector & really, vector & predict, std::vector & train_data, cv::Mat & train_label, std::vector & test_data, cv::Mat & test_label, int train_num, int test_num)
{
	int length = predict.size();
	cout << "train image is :" << length << endl;
	int n = 0;
	int featureNum = 0;
	int cols = 68 * 2;
	for (int i = 0; i < length; i++)
	{
		Mat img = imread(base + predict[i].fileName, CV_LOAD_IMAGE_COLOR);
		Mat patch[68];
		vector rect;
		for (int j = 0; j < 68; j++)
		{
			Rect re;
			re.x = predict[i].points[j].x;
			re.y = predict[i].points[j].y;
			re.x = re.x - 8;
			re.y = re.y - 8;
			check_xy(re.x, re.y, 96, 96, 16);
			re.width = 16;
			re.height = 16;
			Mat roi = img(re);
			patch[j] = roi;
			rect.push_back(re);
		}
		if (i < train_num)
		{
			if (i == 0)
			{
				train_label = Mat::zeros(train_num, cols, CV_32FC1); //注意其中训练和自动训练的接口，还有labelMat一定要用CV_32SC1的类型
			}
			for (int j = 0; j < cols / 2; j++)
			{
				train_label.at(i, 2 * j) = really[i].points[j].x - rect[j].x;//predict[i].points[j].x;
				train_label.at(i, 2 * j + 1) = really[i].points[j].y - rect[j].y;//predict[i].points[j].y;
				//cout << train_label.at(i, 2 * j) << endl;
			}
			/*Mat patch1, patch2;
			for (int i = 0; i < 17; i++)
			{
				for (int j = 0; j < 3; j++)
				{
					if (j == 0)
					{
						hconcat(patch[i][j], patch[i][j + 1], patch1);
					}
					else {
						hconcat(patch1, patch[i][j + 1], patch1);
					}
				}
				if (i == 0)
				{
					patch2 = patch1;
				}
				else if (i > 0)
				{
					vconcat(patch1, patch2, patch2);
				}
			}*/
			Mat dst;
			merge(patch, 68, dst);
			train_data.push_back(dst);
		}
		else
		{
			if (i == train_num)
			{
				test_label = Mat::zeros(test_num, cols, CV_32FC1); //注意其中训练和自动训练的接口，还有labelMat一定要用CV_32SC1的类型
			}
			for (int j = 0; j < cols / 2; j++)
			{
				test_label.at(i - train_num, 2 * j) = really[i - train_num].points[j].x - predict[i - train_num].points[j].x;
				test_label.at(i - train_num, 2 * j + 1) = really[i - train_num].points[j].y - predict[i - train_num].points[j].y;
			}
			Mat dst;
			merge(patch, 68, dst);
			test_data.push_back(dst);
		}
	}
}

void DataPrepareUtil::switch68to5landmark(std::vector & data_68, vector & data_5)
{
	int four_patch[] = { 36,42, 42,48, 30, 48, 54 };
	int length = data_68.size();
	for (int i = 0; i < length; i++)
	{
		LandMark landmark;
		landmark.fileName = data_68[i].fileName;
		for (int j = 0; j < LANDMARK_NUM; j++)
		{
			float x = 0, y = 0;
			if (j >= four_patch[0] && j < four_patch[1])
			{
				landmark.points[0].x += data_68[i].points[j].x;
				landmark.points[0].y += data_68[i].points[j].y;
			}
			if (j >= four_patch[2] && j < four_patch[3])
			{
				landmark.points[1].x += data_68[i].points[j].x;
				landmark.points[1].y += data_68[i].points[j].y;
			}
			if (j == four_patch[4])
			{
				landmark.points[2].x += data_68[i].points[j].x;
				landmark.points[2].y += data_68[i].points[j].y;
			}
			if (j == four_patch[5])
			{
				landmark.points[3].x += data_68[i].points[j].x;
				landmark.points[3].y += data_68[i].points[j].y;
			}
			if (j == four_patch[6])
			{
				landmark.points[4].x += data_68[i].points[j].x;
				landmark.points[4].y += data_68[i].points[j].y;
			}
		}
		for (int k = 0; k < 2; k++)
		{
			landmark.points[k].x = landmark.points[k].x / 6.0;
			landmark.points[k].y = landmark.points[k].y / 6.0;
		}
		data_5.push_back(landmark);
	}
}

void DataPrepareUtil::switchSizePoint(vector & data, int mark_num, cv::Size input_size, cv::Size out_size)
{
	int length = data.size();
	for (int i = 0; i < length; i++)
	{
		for (int j = 0; j < mark_num; j++)
		{
			data[i].points[j].x = data[i].points[j].x * ((float)out_size.width / input_size.width);
			data[i].points[j].y = data[i].points[j].y * ((float)out_size.height / input_size.height);
		}
	}
}

void DataPrepareUtil::writePointVisibletoFile(std::string filePath, vector & data, int landmark)
{
	if (filePath == "" || data.size() == 0)
	{
		cout << " filePath == "" || data.size() == 0 " << endl;
		return;
	}
	ofstream in;
	in.open(filePath, ios::app);
	int length = data.size();
	for (int i = 0; i < length; i++)
	{
		string dataline = data[i].fileName;
		dataline.append(" ");
		for (int j = 0; j < landmark; j++)
		{
			dataline.append(to_string(data[i].points[j].x));
			dataline.append(" ");
			dataline.append(to_string(data[i].points[j].y));
			//if (j != (landmark - 1))
			//{
			dataline.append(" ");
			//}
		}
		for (int j = 0; j < landmark; j++)
		{
			dataline.append(to_string(data[i].visiable[j]< 0.5 ? 0 : 1));
			if (j != (landmark - 1))
			{
				dataline.append(" ");
			}
		}
		in << dataline << "\n";
	}
	in.close();
}

svm_parameter param;
void init_param()
{
	param.svm_type = EPSILON_SVR;
	param.kernel_type = RBF;
	param.degree = 3;
	param.gamma = 0.01;
	param.coef0 = 0;
	param.nu = 0.5;
	param.cache_size = 1000;
	param.C = 20;
	param.eps = 1e-6;
	param.shrinking = 1;
	param.probability = 0;
	param.nr_weight = 0;
	param.weight_label = NULL;
	param.weight = NULL;
}
//void DataPrepareUtil::libSVM_Train(cv::Mat & data, cv::Mat &label, string save_path)
//{
//	init_param();
//	int rows = data.rows;
//	int cols = data.cols;
//
//	svm_problem prob;
//	prob.l = rows;
//
//	svm_node *x_space = new svm_node[(cols + 1)*prob.l];//样本特征存储空间
//	prob.x = new svm_node *[prob.l]; //每一个X指向一个样本
//	cout << "size :" << sizeof(x_space) << endl;
//	prob.y = new double[prob.l];
//
//	//libsvm train data prepare.
//	for (int i = 0; i < rows; i++)
//	{
//		for (int j = 0; j < cols + 1; j++)
//		{
//			if (j == cols)
//			{
//				x_space[i*(cols + 1) + j].index = -1;
//				prob.x[i] = &x_space[i * (cols + 1)];
//				prob.y[i] = label.at(i, 0);
//				break;
//			}
//			x_space[i*(cols + 1) + j].index = j + 1;
//			x_space[i*(cols + 1) + j].value = data.at(i, j);
//		}
//	}
//	cout << "start train svm." << endl;
//	svm_model *model = svm_train(&prob, ¶m);
//
//	cout << "save model" << endl;
//	svm_save_model(save_path.c_str(), model);
//	cout << "done!" << endl;
//
//	delete[] x_space;
//	delete[] prob.x;
//	delete[] prob.y;
//}
//void DataPrepareUtil::libSVM_Predict(std::string mode, cv::Mat & data, cv::Mat &label)
//{
//	svm_model* model = svm_load_model(mode.c_str());
//	int test_cols = data.cols;
//	int test_rows = data.rows;
//	svm_node *test_space = new svm_node[test_cols + 1];
//	//svm_problem prob_test;
//	//libsvm test data prepare.
//	int error = 0;
//	double t0 = (double)cvGetTickCount();
//	for (int i = 0; i < test_rows; i++)
//	{
//		for (int j = 0; j < test_cols + 1; j++)
//		{
//			if (j == test_cols)
//			{
//				test_space[j].index = -1;
//				break;
//			}
//			test_space[j].index = j + 1;
//			test_space[j].value = data.at(i, j);
//		}
//		int d = svm_predict(model, test_space);
//		if (d != label.at(i, 0))
//		{
//			cout << "predict is :" << d << " really is :" << label.at(i, 0) << endl;
//			error++;
//		}
//	}
//	double t1 = (double)cvGetTickCount();
//	cout << "average time is: " << ((t1 - t0) / ((double)cvGetTickFrequency() * 1000 * 1000))*1000.0 / test_rows << "ms" << endl;
//	cout << "acurcy is :" << (float)(test_rows - error) / test_rows << endl;
//	delete[] test_space;
//}

 

#include 
#include "../CascadRegression/MCLC.h"
#include 
#include "../CascadRegression/util/Util.h"
#include 
#include 

#include 
using namespace caffe;
using namespace cv;
using namespace glasssix;
using namespace std;

#define DEVICE 0
#define RESIZE 96
#define IMAGESIZE 96
#define NEED_LANDMARK 5


double predict(Mat &img, MCLC & mclc, int net_id, LandMark & landmark, int mark_num)
{
	double time = 0;
	double t0 = (double)cvGetTickCount();
	vector imgdata;
	imgdata.push_back(img);
	unordered_map result = mclc.Forward(imgdata, net_id);
	double t1 = (double)cvGetTickCount();
	time = (t1 - t0) / ((double)cvGetTickFrequency() * 1000);
	//cout << "time is:" << time << " ms" << endl;
	for (unordered_map::iterator iter = result.begin(); iter != result.end(); iter++) {
		string key = iter->first;
		DataBlob result1 = iter->second;
		vector mt = result1.size;
		if (result1.name == "ip2")
		{
			for (int i = 0; i < mark_num; i++)
			{
				float x = *(result1.data);
				float y = *(++result1.data);
				landmark.points[i].x = x;
				landmark.points[i].y = y;
				(result1.data)++;
			}
		}
		else if (result1.name == "ip3")
		{
			for (int i = 0; i < mark_num; i++)
			{
				landmark.visiable[i] = *(result1.data);
				(result1.data)++;
			}
		}
	}
	return time;
}

void showError(vector &label, vector & predictRe, int mark_num)
{
	int size = label.size();
	float ave_error = 0, eye_error = 0;
	cout << label.size() << "   " << predictRe.size() << endl;
	for (int i = 0; i < size; i++)
	{
		float sum = 0, sum_eye = 0;
		for (int j = 0; j < mark_num; j++)
		{
			if (j == 0)
			{
				sum_eye += pow((label[i].points[j].x - predictRe[i].points[j].x), 2) + pow((label[i].points[j].y - predictRe[i].points[j].y), 2);
			}
			sum += pow((label[i].points[j].x - predictRe[i].points[j].x), 2) + pow((label[i].points[j].y - predictRe[i].points[j].y), 2);
		}
		eye_error += sum_eye;
		float distance = sqrt(pow((label[i].points[0].x - label[i].points[1].x), 2) + pow((label[i].points[0].y - label[i].points[1].y), 2));
		ave_error += 1 / distance * sqrt(sum);
	}
	cout << "eye-error is:" << eye_error / size << endl;
	cout << "all error is:" << ave_error / size << endl;
}
void showVisibleError(vector &label, vector & predictRe, int mark_num)
{
	int size = label.size();
	float error1 = 0, error2 = 0;
	for (int i = 0; i < size; i++)
	{
		for (int j = 0; j < mark_num; j++)
		{
			if (label[i].visiable[j] == 0)
			{
				error2++;
				if (predictRe[i].visiable[j] > 0.5)
				{
					error1++;
				}
			}
		}
	}
	cout << "visiable error is:" << error1 / error2 << endl;
}

void switchPointSize(DataPrepareUtil &dpu)
{
	//switch point size
	string shutter_train = "D:\\face_new\\landmark_5\\shutter_train.txt";
	string shutter_test = "D:\\face_new\\landmark_5\\shutter_test.txt";
	string shutter_val = "D:\\face_new\\landmark_5\\shutter_val.txt";
	vector data_96, data_96_test, data_96_val;
	data_96 = dpu.readStageOneData(shutter_train, NEED_LANDMARK);
	data_96_test = dpu.readStageOneData(shutter_test, NEED_LANDMARK);
	data_96_val = dpu.readStageOneData(shutter_val, NEED_LANDMARK);
	dpu.switchSizePoint(data_96, NEED_LANDMARK, Size(96, 96), Size(128, 128));
	dpu.switchSizePoint(data_96_test, NEED_LANDMARK, Size(96, 96), Size(128, 128));
	dpu.switchSizePoint(data_96_val, NEED_LANDMARK, Size(96, 96), Size(128, 128));

	/*for (int i = 0; i < 10; i++)
	{
	Mat img = imread("D:\\face_new\\landmark_5\\result_img_128\\" + data_96[i].fileName);
	for (int j = 0; j < NEED_LANDMARK; j++)
	{
	circle(img, Point(data_96[i].points[j]), 2, Scalar(255, 0, 0));
	}
	imshow("img", img);
	waitKey(0);
	}*/
	dpu.writeDatatoFile("D:\\face_new\\landmark_5\\result_img_128\\shutter_train.txt", data_96, NEED_LANDMARK);
	dpu.writeDatatoFile("D:\\face_new\\landmark_5\\result_img_128\\shutter_test.txt", data_96_test, NEED_LANDMARK);
	dpu.writeDatatoFile("D:\\face_new\\landmark_5\\result_img_128\\shutter_val.txt", data_96_val, NEED_LANDMARK);
}

void Test300WError(DataPrepareUtil &dpu, MCLC &mclc, int id_68)
{
	// 300W 错误 
	string img_300w = "D:\\face\\face_alignment\\300W_resize\\";
	string text_300w = "D:\\face\\face_alignment\\300W_resize\\result_point_96_68.txt";
	double time1 = 0;
	vector data_68;
	data_68 = dpu.readStageOneData(text_300w, LANDMARK_NUM);
	vector data_5;
	vector pre_data_5;
	dpu.switch68to5landmark(data_68, data_5);
	int size = data_5.size();
	for (int j = 0; j < size; j++)
	{
		LandMark landmark;
		landmark.fileName = data_5[j].fileName;
		Mat img = imread(img_300w + data_5[j].fileName);
		time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);

		pre_data_5.push_back(landmark);
	}
	cout << "300W test result" << endl;
	showError(data_5, pre_data_5, NEED_LANDMARK);
	cout << "average time is :" << time1 / size << endl;
	//dpu.writeDatatoFile("D:\\face\\face_alignment\\300W_resize\\result_point_96_5.txt", data_5, NEED_LANDMARK);
}

void L2LossPre(vector &preRe)
{
	string base = "E:\\work\\face_alignment\\lanmark_5\\model\\regression_96_5\\";
	string prototext = base + "test_5_op.prototxt";
	string mode_path = base + "save_path\\_iter_150000.caffemodel";
	string img_path = "D:\\face_new\\landmark_5\\result_img_96\\";
	string text_label = img_path + "shutter_test.txt";

	MCLC mclc;
	int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
	DataPrepareUtil dpu;
	vector data;
	data = dpu.readStageOneData(text_label, NEED_LANDMARK);
	cout << "data length is:" << data.size() << endl;
	int length = data.size();

	double time1 = 0;
	for (int i = 0; i < length; i++)
	{
		LandMark landmark;
		landmark.fileName = data[i].fileName;
		Mat img = imread(img_path + data[i].fileName);

		time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);

		preRe.push_back(landmark);

		//for (int j = 0; j < NEED_LANDMARK; j++)
		//{
		//	circle(img, Point(landmark.points[j]), 2, Scalar(255, 0, 0));
		//}
		//imshow("img", img);
		//waitKey(0);
	}
	cout << "average time1 is:" << time1 / length << " ms" << endl;
	showError(data, preRe, NEED_LANDMARK);
}

void absLossPre(string img_path, string text_label, MCLC &mclc, vector &preRe, DataPrepareUtil &dpu, vector & data)
{
	string base = "E:\\work\\face_alignment\\lanmark_5\\model\\umd_96_5\\";
	string prototext = base + "test_5_op_change_abs.prototxt";
	string mode_path = base + "save_path_abs\\_iter_200000.caffemodel";

	int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
	data = dpu.readPointVisibleData(text_label, NEED_LANDMARK);
	cout << "data length is:" << data.size() << endl;
	int length = data.size();

	double time1 = 0;
	for (int i = 0; i < length; i++)
	{
		LandMark landmark;
		landmark.fileName = data[i].fileName;
		Mat img = imread(img_path + data[i].fileName);

		time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);

		preRe.push_back(landmark);

	}
	cout << "average time1 is:" << time1 / length << " ms" << endl;
	showError(data, preRe, NEED_LANDMARK);
}

void pointVisibleLossPre(string img_path, string text_label, MCLC &mclc, vector &preRe, DataPrepareUtil &dpu, vector & data)
{
	string base = "E:\\work\\face_alignment\\lanmark_5\\model\\umd_128_5\\";
	string prototext = base + "test_5_op_loss.prototxt";
	string mode_path = base + "save_path1\\_iter_200000.caffemodel";

	int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
	data = dpu.readPointVisibleData(text_label, NEED_LANDMARK);
	cout << "data length is:" << data.size() << endl;
	int length = data.size();

	double time1 = 0;
	for (int i = 0; i < length; i++)
	{
		LandMark landmark;
		landmark.fileName = data[i].fileName;
		Mat img = imread(img_path + data[i].fileName);

		time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);

		preRe.push_back(landmark);

		//for (int j = 0; j < NEED_LANDMARK; j++)
		//{
		//	Scalar scalar;
		//	if (landmark.visiable[j] < 0.5)
		//	{
		//		scalar = Scalar(255, 0, 0);
		//	}
		//	else {
		//		scalar = Scalar(0, 0, 255);
		//	}
		//	circle(img, Point(landmark.points[j]), 2, scalar, -1);
		//}
		//imshow("img", img);
		//waitKey(0);
	}
	cout << "average time1 is:" << time1 / length << " ms" << endl;
	showError(data, preRe, NEED_LANDMARK);
}

void smoothLossPre(string img_path, string text_label, MCLC &mclc, vector &preRe, DataPrepareUtil &dpu, vector & data)
{
	string base = "E:\\work\\face_alignment\\lanmark_5\\model\\regression_96_5\\";
	string prototext = base + "test_5_op_loss.prototxt";
	string mode_path = base + "save_path_smooth\\_iter_200000.caffemodel";

	int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
	data = dpu.readStageOneData(text_label, NEED_LANDMARK);
	cout << "data length is:" << data.size() << endl;
	int length = data.size();

	double time1 = 0;
	for (int i = 0; i < length; i++)
	{
		LandMark landmark;
		landmark.fileName = data[i].fileName;
		Mat img = imread(img_path + data[i].fileName);

		time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);

		preRe.push_back(landmark);

	}
	cout << "average time1 is:" << time1 / length << " ms" << endl;
	showError(data, preRe, NEED_LANDMARK);
}

void size64Pre()
{
	string base = "E:\\work\\face_alignment\\lanmark_5\\model\\regression_64_5\\";
	string prototext = base + "test_5_op.prototxt";
	string mode_path = base + "save_path\\_iter_150000.caffemodel";
	string img_path = "D:\\face_new\\landmark_5\\result_img_64\\";
	string text_label = img_path + "shutter_test.txt";

	MCLC mclc;
	int id_68 = mclc.AddNet(prototext, mode_path, DEVICE);
	DataPrepareUtil dpu;
	vector data;
	data = dpu.readStageOneData(text_label, NEED_LANDMARK);
	cout << "data length is:" << data.size() << endl;
	int length = data.size();
	vector preRe;

	double time1 = 0;
	for (int i = 0; i < length; i++)
	{
		LandMark landmark;
		landmark.fileName = data[i].fileName;
		Mat img = imread(img_path + data[i].fileName);

		time1 += predict(img, mclc, id_68, landmark, NEED_LANDMARK);

		preRe.push_back(landmark);

		//for (int j = 0; j < NEED_LANDMARK; j++)
		//{
		//	circle(img, Point(landmark.points[j]), 2, Scalar(255, 0, 0));
		//}
		//imshow("img", img);
		//waitKey(0);
	}
	cout << "average time1 is:" << time1 / length << " ms" << endl;
	showError(data, preRe, NEED_LANDMARK);
}

//size: brow, eye, nose, mouth, face 
const float four_size[] = { 24,24, 28,28, 48,32 };
string mark_4_img_dir[] = { "eye\\", "nose\\", "mouth\\" };
string mark_4_img_train[] = { "eye_train.txt", "nose_train.txt", "mouth_train.txt" };
string mark_4_img_test[] = { "eye_test.txt", "nose_test.txt", "mouth_test.txt" };
void getRect(LandMark landmark, int flag, vector &rects, int cols, int rows)
{
	Rect rect, rect1;
	Point p1;
	if (flag == 0)
	{
		p1 = landmark.points[0];
	}
	else if (flag == 1)
	{
		p1 = landmark.points[2];
	}
	else if (flag == 2)
	{
		p1 = landmark.points[3];
	}

	float x, y, width, height;
	rect.width = four_size[2 * flag];
	rect.height = four_size[2 * flag + 1];

	x = p1.x;
	y = p1.y;
	rect.x = min(max(x - four_size[2 * flag] * 0.5f, 0.0f), (float)(IMAGESIZE - four_size[2 * flag]));
	rect.y = min(max(y - four_size[2 * flag + 1] * 0.5f, 0.0f), (float)(IMAGESIZE - four_size[2 * flag + 1]));

	rects.push_back(rect);
}
void resetRect(Rect & rect, int cols, int rows)
{
	if (rect.x < 0)
	{
		rect.x = 0;
	}
	if (rect.y < 0)
	{
		rect.y = 0;
	}
	if (rect.x + rect.width > cols)
	{
		rect.x = cols - rect.width;
	}
	if (rect.y + rect.height > rows)
	{
		rect.y = rows - rect.height;
	}
}
void addRect(vector & rects)
{
	srand((unsigned)time(NULL));
	int dir[] = { 1,1, 1,-1, 1,0, -1,1, -1,-1, -1,0, 0,1, 0,-1 };
	if (rects.size() == 1)
	{
		for (int i = 0; i < 8; i++)
		{
			Rect r(rects[0]);
			r.x += dir[2 * i] * (rand() % 4 + 1);
			r.y += dir[2 * i + 1] * (rand() % 4 + 1);
			resetRect(r, IMAGESIZE, IMAGESIZE);
			rects.push_back(r);
		}
	}
}
void createPatchFile(string base, vector &data, vector &preRe, DataPrepareUtil &dpu)
{
	const int path_size = 1;
	vector landmark_four[path_size][2];
	int istd = 0;

	for (int j = 0; j < path_size; j++)
	{
		dpu.clearFileData(base + mark_4_img_dir[j] + mark_4_img_train[j]);
		dpu.clearFileData(base + mark_4_img_dir[j] + mark_4_img_test[j]);
	}

	int length = preRe.size();
	for (int i = 0; i < length; i++)
	{
		if (i >= 12000)
		{
			istd = 1;
		}
		for (int j = 0; j < path_size; j++)
		{
			Mat img = imread(base + preRe[i].fileName);
			vector rects;
			getRect(preRe[i], j, rects, img.cols, img.rows);
			//cout << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;
			addRect(rects);
			for (int n = 0; n < rects.size(); n++)
			{
				Mat roi = img(rects[n]);

				string dir = base + mark_4_img_dir[j];
				if (_access(dir.c_str(), 0) == -1)
				{
					_mkdir(dir.c_str());
				}
				imwrite(dir + to_string(n) + "_" + preRe[i].fileName, roi);
				//坐标转换
				LandMark mark;
				mark.fileName = to_string(n) + "_" + preRe[i].fileName;

				float offsetX = (data[i].points[0].x - rects[n].x) * four_size[2 * j] / rects[n].width;
				float offsetY = (data[i].points[0].y - rects[n].y) * four_size[2 * j + 1] / rects[n].height;
				mark.points[0].x = offsetX;
				mark.points[0].y = offsetY;
				landmark_four[j][istd].push_back(mark);
			}
		}

		if (landmark_four[0][0].size() > 1000)
		{
			for (int j = 0; j < path_size; j++)
			{
				dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][0], 1);
				(landmark_four[j][0]).clear();
			}
		}
		if (landmark_four[0][1].size() > 1000)
		{
			for (int j = 0; j < path_size; j++)
			{
				dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_test[j], landmark_four[j][1], 1);
				(landmark_four[j][1]).clear();
			}
		}
	}

	for (int j = 0; j < path_size; j++)
	{
		dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][0], 1);
		增加验证集数据 数据平衡
		//dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][1], mark_num[j]);
		//dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][1], mark_num[j]);
		//dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_train[j], landmark_four[j][1], mark_num[j]);
		
		dpu.writeDatatoFile(base + mark_4_img_dir[j] + mark_4_img_test[j], landmark_four[j][1], 1);
	}
}
void stageTwoPredict(string img_path, MCLC &mclc, vector &label, vector &preRe)
{
	string base = "E:\\work\\face_alignment\\lanmark_5\\model\\regression_stage_two\\eye\\";
	string prototext_eye = base + "test_eye_op.prototxt";
	string mode_path_eye = base + "save_path\\_iter_150000.caffemodel";
	int id_eye = mclc.AddNet(prototext_eye, mode_path_eye, DEVICE);

	double time_four = 0;
	int length = preRe.size();
	for (int i = 0; i < length; i++)
	{
		LandMark landmark;
		landmark.fileName = preRe[i].fileName;
		Mat img = imread(img_path + preRe[i].fileName);

		vector rects;
		getRect(preRe[i], 0, rects, img.cols, img.rows);

		Mat roi = img(rects[0]);
		time_four += predict(roi, mclc, id_eye, landmark, 1);
		preRe[i].points[0].x = landmark.points[0].x * (rects[0].width / four_size[0]) + rects[0].x;
		preRe[i].points[0].y = landmark.points[0].y * (rects[0].height / four_size[1]) + rects[0].y;
	}
	showError(label, preRe, NEED_LANDMARK);
	cout << "stage two ........................................................." << endl;
	cout << "average time is :" << time_four / length << " ms" << endl;
}

void showMeanShape()
{
	string shape_text = "D:\\face_new\\landmark_5\\result_img_128\\landmark_128_5.txt";
	vectordata;
	DataPrepareUtil dpu;
	data = dpu.readStageOneData(shape_text, NEED_LANDMARK);
	cout << "data length is:" << data.size() << endl;

	LandMark landmark;
	int length = data.size();
	for (int i = 0; i < length; i++)
	{
		for (int j = 0; j < NEED_LANDMARK; j++)
		{
			landmark.points[j].x += data[i].points[j].x;
			landmark.points[j].y += data[i].points[j].y;
		}
	}
	for (int j = 0; j < NEED_LANDMARK; j++)
	{
		landmark.points[j].x = landmark.points[j].x / length;
		landmark.points[j].y = landmark.points[j].y / length;
		cout << "shape j:" << j << "  ( " << landmark.points[j].x << " , " << landmark.points[j].y << " )" << endl;
	}
}

int main(int argc, const char** argv) {

	//L2LossPre();
	//AbsLossPre();
	//size64Pre();

	DataPrepareUtil dpu;
	vector preRe;
	vector data;

	string img_path = "D:\\UMD\\result_img_96\\";
	string text_label = img_path + "shutter_96_5_test.txt";
	MCLC mclc;

	//showMeanShape();
	absLossPre(img_path, text_label, mclc, preRe, dpu, data);
	//smoothLossPre(img_path, text_label, mclc, preRe, dpu, data);
	//string base = "D:\\face_new\\landmark_5\\result_img_96\\";


	//pointVisibleLossPre(img_path, text_label, mclc, preRe, dpu, data);
	//showVisibleError(data, preRe, NEED_LANDMARK);

	std::system("PAUSE");
	return 0;
}

Prepare data

#include "../data300w/util/Util.h"
#include "cunpd.hpp"
#include 
#include 
#include 
#include   
#include  

using namespace std;
using namespace cv;
using namespace glasssix;

#define ZOOM 1.0
#define RESIZE 96

#define NEED_LANDMARK 5
#define CURRENT_LANDMARK 21
string search_base = "H:\\UMD\\";
string search_dir_[] = { search_base + "umdfaces_batch1", search_base + "umdfaces_batch2", search_base + "umdfaces_batch3"};

string text_file[] = { search_base +"umdfaces_batch1\\umdfaces_batch1_ultraface.csv", search_base + "umdfaces_batch2\\umdfaces_batch2_ultraface.csv", search_base + "umdfaces_batch3\\umdfaces_batch3_ultraface.csv" };
string text_pre[] = { "batch1_", "batch2_", "batch3_"};
string tail_[] = { ".jpg", ".jpg" , ".jpg" };

string base = search_base + "landmark_5\\";
string result_img = base + "result_img4_" + to_string(RESIZE) + "\\";
string result_txt = base + "landmark4_" + to_string(RESIZE) + "_5.txt";
const int five_point[5] = { 7, 10, 14, 17, 19};

int main11() {
	// prepare data code.
	cunpd *pd = new cunpd();

	vector result_five_point;
	int model_id = pd->AddNpdModel(0);
	int faceNum = 0;
	DataPrepareUtil dpUtil;
	dpUtil.clearFileData(result_txt);

	string base_dir = base;
	if (_access(base_dir.c_str(), 0) == -1)
	{
		_mkdir(base_dir.c_str());
	}

	long count = 0;
	for (int temp = 0; temp < 3; temp++)
	{
		long countNum = 0;
		vector result_point;
		dpUtil.readFileData(text_file[temp], result_point, CURRENT_LANDMARK);

		for (vector::iterator itor = result_point.begin(); itor != result_point.end();)
		{
			if (count % 500 == 0)
			{
				cout << "prepare count is: " << count << " current img path is :" << result_point[0].fileName << endl;
			}
			count++;

			Mat img;
			int widths = 0, heights = 0;
			Mat dstImg;
			try {

				string img_path = search_dir_[temp] + "\\" + result_point[0].fileName;
				if (_access(img_path.c_str(), 0) == -1)
				{
					cout << "coun't found filename" << search_dir_[temp] + result_point[0].fileName << endl;
					result_point.erase(itor);
					continue;
				}

				img = imread(search_dir_[temp] + "\\" + result_point[0].fileName);

				widths = img.cols;
				heights = img.rows;
				dstImg.create(heights, widths, CV_8UC1);
				cvtColor(img, dstImg, CV_BGR2GRAY);
			}
			catch (Exception e)
			{
				cout << "exception:" << " current file:" << result_point[0].fileName << endl;
				system("PAUSE");
			}

			//for (int k = 0; k < CURRENT_LANDMARK; k++)
			//{
			//	circle(img, Point(result_point[0].points[k]), 2, Scalar(255, 0, 0));
			//	putText(img, to_string(k), Point(Point(result_point[0].points[k])), 0.3, 0.3, Scalar(255, 0, 0));	
			//}
			//imshow("img", img);
			//waitKey(0);

			vector  face = pd->detect(dstImg, model_id, 48);
			LandMark landmark;
			int length = face.size();
			if (length == 0)
			{
				cout << "not found face ." << endl;
			}
			for (int j = 0; j < length; j++)
			{
				if (face[j].score > 15)
				{
					Rect rect = face[j].rect;

					if (result_point[0].points[0].x > rect.x && result_point[0].points[0].x < rect.x + rect.width
						&& result_point[0].points[0].y > rect.y && result_point[0].points[0].y < rect.y + rect.height
						&&result_point[0].points[12].x > rect.x && result_point[0].points[12].x < rect.x + rect.width
						&& result_point[0].points[12].y > rect.y && result_point[0].points[12].y < rect.y + rect.height
						&&result_point[0].points[16].x > rect.x && result_point[0].points[16].x < rect.x + rect.width
						&& result_point[0].points[16].y > rect.y && result_point[0].points[16].y < rect.y + rect.height
						&&result_point[0].points[20].x > rect.x && result_point[0].points[20].x < rect.x + rect.width
						&& result_point[0].points[20].y > rect.y && result_point[0].points[20].y < rect.y + rect.height
						&& (abs(result_point[0].points[7].y - result_point[0].points[17].y) > (rect.height / 6.0)))
					{
						int rect_w = rect.width;
						int rect_h = rect.height;
						rect.width = rect_w * ZOOM;
						rect.height = rect_h * ZOOM;
						rect.x = max(rect.x - (ZOOM - 1.0) * rect_w / 2.0, 0.0);
						rect.y = max(rect.y - (ZOOM - 1.0) * rect_h / 2.0, 0.0);

						if (rect.x + rect.width > widths)
						{
							rect.width = widths - rect.x;
							//rect.height = rect_h * rect.width / rect_w;
						}
						if (rect.y + rect.height > heights)
						{
							rect.height = heights - rect.y;
							//rect.width = rect_w * rect.height / rect_h;
						}

						Mat roi_face, result_face;
						roi_face = img(rect);
						result_face.create(RESIZE, RESIZE, CV_8UC3);
						cv::resize(roi_face, result_face, Size(RESIZE, RESIZE));

						string dir = result_img;
						if (_access(dir.c_str(), 0) == -1)
						{
							_mkdir(dir.c_str());
						}
						imwrite(dir + text_pre[temp] + to_string(countNum) + ".png", result_face);

						landmark.fileName = text_pre[temp] + to_string(countNum) + ".png";

						countNum++;
						//坐标转换
						//double t0 = (double)cvGetTickCount();
						//#pragma omp parallel for num_threads(4)
						for (int k = 0; k < NEED_LANDMARK; k++)
						{
							landmark.visible[k] = result_point[0].visible[five_point[k]];
							if (result_point[0].points[five_point[k]].x == -1 || result_point[0].points[five_point[k]].y == -1) {
								cout << "this points is NULL" << endl;
								continue;
							}
							landmark.points[k].x = ((float)RESIZE / rect.width) * (result_point[0].points[five_point[k]].x - rect.x);
							landmark.points[k].y = ((float)RESIZE / rect.height) * (result_point[0].points[five_point[k]].y - rect.y);
						}
						faceNum++;
						result_five_point.push_back(landmark);

						if (result_five_point.size() >= 1000)
						{
							dpUtil.writePointVisibletoFile(result_txt, result_five_point, NEED_LANDMARK);
							result_five_point.clear();
						}

						//for (int k = 0; k < 5; k++)
						//{
						//	circle(result_face, Point(landmark.points[k]), 2, Scalar(255, 0, 0));
						//}
						//imshow("img", result_face);
						//waitKey(0);

						break;
					}
				}
			}
			result_point.erase(itor);
		}
	}

	if (result_five_point.size() > 0)
	{
		dpUtil.writePointVisibletoFile(result_txt, result_five_point, NEED_LANDMARK);
		result_five_point.clear();
	}

	std::system("PAUSE");
	delete pd;

	return 0;
}

 

 

 

Argument data

#include 
#include 
#include "../data300w/util/Util.h"
#include "cunpd.hpp"
#include 
#include 
#include   
#include 
#include 
#include 
#include 

using namespace std;
using namespace cv;
using namespace glasssix;

#define ZOOM_ 1.0
#define SIZE 96

extern void  splitString(const string& s, vector& v, const string& c);

template 
Type stringToNum(const string& str)
{
	istringstream iss(str);
	Type num;
	iss >> num;
	return num;
}
void writeHistoFile(std::string filePath, vector & densi_data)
{
	if (filePath == "" || densi_data.size() == 0)
	{
		return;
	}
	ofstream in;
	in.open(filePath, ios::app);   //ios::trunc
	int length = densi_data.size();
	for (int i = 0; i < length; i++)
	{
		string dataline = to_string(densi_data[i]);
		in << dataline << "\n";
	}
	in.close();
}

float getHorizontal(LandMark &landmark)
{
	float tan_theta = (landmark.points[1].y - landmark.points[0].y) / (landmark.points[1].x - landmark.points[0].x);
	float theta = atan(tan_theta);
	return theta * 180 / 3.1415926;
}
void showHistogram(vector & hor_data)
{
	int densi[60] = { 0 };
	int length = hor_data.size();
	for (int i = 0; i < length; i++)
	{
		if (floor((hor_data[i] + 30)) >= 0 && floor((hor_data[i] + 30)) < 60)
		{
			densi[(int)floor((hor_data[i] + 30))]++;
		}

		if (floor((hor_data[i] + 30)) < 0)
		{
			densi[0]++;
		}
		else if (floor((hor_data[i] + 30)) >= 60)
		{
			densi[60]++;
		}
	}
	string density_text = "D:\\UMD\\density_text.txt";
	vectordensity_data(densi, densi + 60);
	writeHistoFile(density_text, density_data);

	Mat histImg;
	histImg.create(1000, 1600, CV_8UC3);
	histImg.setTo(0);
	int offset = 10;
	for (int i = 0; i < 60; i++)
	{
		double tmpCount = densi[i];
		rectangle(histImg, Point2f(offset + i * 25, 1000), Point2f(offset + i * 25, 1000 - tmpCount / 15.0), Scalar::all(255), -1);  //画出直方图  
		putText(histImg, to_string(i - 29), Point2f(offset + i * 25 + 3, 1000 - 3), 0.3, 0.3, Scalar(0, 0, 255));
		Point2f pt0;
		pt0.x = offset + i * 25;
		pt0.y = 1000 - densi[i] / 15.0;

		Point2f pt1;
		pt1.x = offset + (i + 1) * 25;
		pt1.y = 1000 - densi[i + 1] / 15.0;
		line(histImg, pt0, pt1, Scalar(255, 0, 0), 1); //连接直方图的顶点  
	}
	imshow("hist", histImg);
	waitKey(0);
}
void getDatahor(string file1, vector & hor_data)
{
	int mark_num = 5;
	DataPrepareUtil dpu;
	vector data;
	dpu.readFileData(file1, data, mark_num);
	int length = data.size();
	for (int i = 0; i < length; i++)
	{
		float hor = getHorizontal(data[i]);
		hor_data.emplace_back(hor);
	}
}

void rotation(float theta, Mat &img, Mat &dst, Size img_size, LandMark &landmark, int mark_num)
{
	//rotation
	Mat mat = img;
	Point2f center(img_size.width / 2, img_size.height / 2);
	double angle = theta;

	Mat rot = getRotationMatrix2D(center, angle, 1);
	Rect bbox = RotatedRect(center, mat.size(), angle).boundingRect();

	cv::warpAffine(mat, dst, rot, bbox.size());

	for (int j = 0; j < mark_num; j++)
	{
		float theta = -3.1415926 / (180 / angle);
		float x1 = landmark.points[j].x - rot.at(1, 2);
		float y1 = landmark.points[j].y - rot.at(0, 2);
		landmark.points[j].x = x1 * cos(theta) - y1 * sin(theta);
		landmark.points[j].y = x1 * sin(theta) + y1 * cos(theta);

		//circle(dst, Point(x, y), 2, Scalar(255, 0, 0));
	}
	//cv::imshow("dst", dst);
	//cv::waitKey(0);
}

void augment_data(string img_path, string img_text, string result_path, string result_text)
{
	DataPrepareUtil dpu;
	int mark_num = 5;
	srand((unsigned)time(NULL));

	vector data;
	dpu.readFileData(img_text, data, mark_num);

	vector rotation_data;
	vector hor_data;
	getDatahor(img_text, hor_data);
	int length = hor_data.size();
	for (int i = 0; i < length; i++)
	{
		if (hor_data[i] > 0 && hor_data[i] < 3)
		{
			Mat dst;
			Mat img = imread(img_path + data[i].fileName);
			LandMark landmark(data[i]);
			rotation(25, img, dst, Size(96, 96), landmark, mark_num);
			rotation_data.push_back(landmark);
		}
	}

}

bool getFaceRect(cunpd &pd, int model_id, Mat &dstImg, LandMark & landmark, Rect & rect)
{
	const int widths = dstImg.cols;
	const int heights = dstImg.rows;
	vector  face = pd.detect(dstImg, model_id, 48);
	int length = face.size();
	if (length == 0)
	{
		cout << "not found face ." << endl;
	}
	for (int j = 0; j < length; j++)
	{
		if (face[j].score > 15)
		{
			rect = face[j].rect;

			if (landmark.points[0].x > rect.x && landmark.points[0].x < rect.x + rect.width
				&& landmark.points[0].y > rect.y && landmark.points[0].y < rect.y + rect.height
				&&landmark.points[12].x > rect.x && landmark.points[12].x < rect.x + rect.width
				&& landmark.points[12].y > rect.y && landmark.points[12].y < rect.y + rect.height
				&&landmark.points[16].x > rect.x && landmark.points[16].x < rect.x + rect.width
				&& landmark.points[16].y > rect.y && landmark.points[16].y < rect.y + rect.height
				&&landmark.points[20].x > rect.x && landmark.points[20].x < rect.x + rect.width
				&& landmark.points[20].y > rect.y && landmark.points[20].y < rect.y + rect.height
				&& (abs(landmark.points[7].y - landmark.points[17].y) > (rect.height / 6.0)))
			{
				int rect_w = rect.width;
				int rect_h = rect.height;
				rect.width = rect_w * ZOOM_;
				rect.height = rect_h * ZOOM_;
				rect.x = max(rect.x - (ZOOM_ - 1.0) * rect_w / 2.0, 0.0);
				rect.y = max(rect.y - (ZOOM_ - 1.0) * rect_h / 2.0, 0.0);

				if (rect.x + rect.width > widths)
				{
					rect.width = widths - rect.x;
				}
				if (rect.y + rect.height > heights)
				{
					rect.height = heights - rect.y;
				}
				return true;
			}
		}
	}
	return false;

}

void getoffsetRect(Rect & rect, vector & all_rect, int cols, int rows, int max_offset)
{
	srand((unsigned)time(NULL));
	Rect rect0(rect), rect1(rect);
	int offsetx = rand() % max_offset + 1;
	int offsety = rand() % max_offset + 1;

	if (rect.x > offsetx && rect.y > offsety)
	{
		rect0.x = rect.x - offsetx;
		rect0.y = rect.y - offsety;
	}

	offsetx = rand() % max_offset + 1;
	offsety = rand() % max_offset + 1;

	if (rect.x + rect.width + offsetx < cols && rect.y + rect.height + offsety < rows)
	{
		rect1.x = rect.x + offsetx;
		rect1.y = rect.y + offsety;
	}
	all_rect.push_back(rect0);
	all_rect.push_back(rect1);
}

#define NEED_LANDMARK 5
#define CURRENT_LANDMARK 21
const int five_points[5] = { 7, 10, 14, 17, 19 };
string search_base = "H:\\UMD\\";
string search_dir_[] = { search_base + "umdfaces_batch1", search_base + "umdfaces_batch2", search_base + "umdfaces_batch3" };

string text_file[] = { search_base + "umdfaces_batch1\\umdfaces_batch1_ultraface.csv", search_base + "umdfaces_batch2\\umdfaces_batch2_ultraface.csv", search_base + "umdfaces_batch3\\umdfaces_batch3_ultraface.csv" };
string text_pre[] = { "batch1_aug_", "batch2_aug_", "batch3_aug_" };
string tail_[] = { ".jpg", ".jpg" , ".jpg" };

string base = search_base + "landmark_5\\augment_img\\";
string result_img = base + "result_img_" + to_string(SIZE) + "\\";
string result_txt = base + "landmark_" + to_string(SIZE) + "_5.txt";
const int theta_offset = 5;
const int theta_max = 20;
vector rotation_point;
int countNum = 0;
bool ready = false;
std::mutex mtx_syn;
std::condition_variable cv_syn;
void roll_yaw_pitch_data(LandMark result_mark, int temp, cunpd &pd, int model_id, DataPrepareUtil &dpu)
{

	float roll = result_mark.direct[2];

	string img_path = search_dir_[temp] + "\\" + result_mark.fileName;
	if (_access(img_path.c_str(), 0) == -1)
	{
		cout << "coun't found filename" << img_path << endl;
		return;
	}
	Mat img = imread(img_path);
	Mat dstImg; //dstImg.create(heights, widths, CV_8UC1);
	cvtColor(img, dstImg, CV_BGR2GRAY);
	//yaw 增强 pitch 增强
	for (int j = 0; j < 2; j++)
	{
		if (result_mark.direct[j] > -theta_offset && result_mark.direct[j] < theta_offset)
		{
			Rect rect;
			LandMark landmark(result_mark);
			bool success = getFaceRect(pd, model_id, dstImg, landmark, rect);
			if (success)
			{
				vector all_rect;
				getoffsetRect(rect, all_rect, img.cols, img.rows, 4);
				for (int i = 0; i < 2; i++)
				{
					LandMark dst_landmark;
					//vector filenames;
					//splitString(landmark.fileName, filenames, "/");
					//string filename = filenames[filenames.size()-1];
					
					std::unique_lock lck(mtx_syn);
					dst_landmark.fileName = text_pre[temp] + to_string(countNum++) + ".png";
					lck.unlock();
					//cout << img.rows << " " << rotat_img.cols << " " << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;

					Mat roi_face = img(all_rect[i]);
					cv::resize(roi_face, roi_face, Size(SIZE, SIZE));
					//坐标转换
					for (int k = 0; k < 5; k++)
					{
						dst_landmark.visible[k] = landmark.visible[five_points[k]];
						dst_landmark.points[k].x = ((float)SIZE / all_rect[i].width) * (landmark.points[five_points[k]].x - all_rect[i].x);
						dst_landmark.points[k].y = ((float)SIZE / all_rect[i].height) * (landmark.points[five_points[k]].y - all_rect[i].y);
					}
					imwrite(result_img + dst_landmark.fileName, roi_face);

					std::unique_lock lck1(mtx_syn);
					rotation_point.push_back(dst_landmark);
					lck1.unlock();
				}
			}

		}
	}
	// roll 增强
	if (roll > -theta_offset && roll < theta_offset)
	{
		for (int i = -1; i < 2; i = i + 2)
		{
			Mat rotat_img;
			LandMark landmark(result_mark);
			int theta = (rand() % theta_max + theta_offset) * i;
			rotation(theta, img, rotat_img, Size(SIZE, SIZE), landmark, CURRENT_LANDMARK);

			Mat dstImg; //dstImg.create(heights, widths, CV_8UC1);
			cvtColor(rotat_img, dstImg, CV_BGR2GRAY);

			//for (int j = 0; j < CURRENT_LANDMARK; j++)
			//{
			//	circle(rotat_img, Point(landmark.points[j]), 2, Scalar(255, 0, 0));
			//}
			//imshow("img", rotat_img);
			//waitKey(0);

			LandMark dst_landmark;

			//vector filenames;
			//splitString(landmark.fileName, filenames, "/");
			//string filename = filenames[filenames.size()-1];
			std::unique_lock lck(mtx_syn);
			dst_landmark.fileName = text_pre[temp] + to_string(countNum++) + ".png";
			lck.unlock();
			Rect rect;
			bool success = getFaceRect(pd, model_id, dstImg, landmark, rect);
			if (success)
			{
				//cout << rotat_img.rows << " " << rotat_img.cols << " " << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;
				Mat roi_face = rotat_img(rect);
				cv::resize(roi_face, roi_face, Size(SIZE, SIZE));
				//坐标转换
				for (int k = 0; k < 5; k++)
				{
					dst_landmark.visible[k] = landmark.visible[five_points[k]];
					dst_landmark.points[k].x = ((float)SIZE / rect.width) * (landmark.points[five_points[k]].x - rect.x);
					dst_landmark.points[k].y = ((float)SIZE / rect.height) * (landmark.points[five_points[k]].y - rect.y);
				}
				imwrite(result_img + dst_landmark.fileName, roi_face);

				std::unique_lock lck(mtx_syn);
				rotation_point.push_back(dst_landmark);

				if (rotation_point.size() > 500)
				{
					dpu.writePointVisibletoFile(result_txt, rotation_point, NEED_LANDMARK);
					rotation_point.clear();
				}
				if (countNum % 500 == 0)
				{
					cout << "prepare data:" << countNum << endl;
				}
				lck.unlock();
			}
		}
	}



}

vector result_point;
void deal_thread(int temp, int model_id, DataPrepareUtil &dpu, cunpd &pd)
{
	while (true)
	{
		std::unique_lock lck(mtx_syn);
		while (!ready) {
			cv_syn.wait(lck);
		}
		//
		auto itor = result_point.begin();
		auto itor2 = result_point.end();
		if (itor == itor2)
		{
			break;
		}
		LandMark landmark(result_point[0]);
		result_point.erase(itor);
//		cout << "landmark.fileName is:"< lck(mtx_syn);
	ready = true;
	cv_syn.notify_all();
}


int main()
{

	cunpd pd;
	int model_id = pd.AddNpdModel(0);

	// 查看数据分布
	//string file1 = "D:\\UMD\\landmark_128_5.txt";
	//vector hor_data;
	//getDatahor(file1, hor_data);
	//showHistogram(hor_data);

	/*string img_path = "D:\\UMD\\result_img_96\\";
	string result_path = "D:\\UMD\\arguement_data\\";
	string img_text = img_path + "shutter_96_5_train.txt";
	string result_text = result_path + "augment_96_5_train.txt";
	augment_data(img_path, img_text, result_path, result_text);*/

	string base_dir = base;
	if (_access(base_dir.c_str(), 0) == -1)
	{
		_mkdir(base_dir.c_str());
	}
	string dir = result_img;
	if (_access(dir.c_str(), 0) == -1)
	{
		_mkdir(dir.c_str());
	}

	srand((unsigned)time(NULL));
	DataPrepareUtil dpUtil;

	dpUtil.clearFileData(result_txt);

	long count = 0;

	vector rotation_point;
	for (int temp = 0; temp < 3; temp++)
	{
		long countNum = 0;
		//vector result_point;

		dpUtil.readFileData(text_file[temp], result_point, CURRENT_LANDMARK);

		std::thread threads[4];
		for (int i = 0; i < 4; i++)
		{
			threads[i] = std::thread(deal_thread, temp, model_id, dpUtil, pd);
			//threads[i] = std::thread(threadA, i, result_point, temp, model_id, dpUtil, pd);
		}
		cout << "temp start:" << temp << endl;
		go();
		for (auto &th : threads) {
			th.join();
		}
		cout << "temp end:" << temp << endl;
		//for (vector::iterator itor = result_point.begin(); itor != result_point.end();)
		//{
		//	if (count % 500 == 0)
		//	{
		//		cout << "prepare count is: " << count << " current img path is :" << result_point[0].fileName << endl;
		//	}
		//	count++;


			/*
			float yaw = result_point[0].direct[0];
			float pitch = result_point[0].direct[1];
			float roll = result_point[0].direct[2];

			string img_path = search_dir_[temp] + "\\" + result_point[0].fileName;
			if (_access(img_path.c_str(), 0) == -1)
			{
				cout << "coun't found filename" << img_path << endl;
				result_point.erase(itor);
				continue;
			}

			Mat img = imread(img_path);
			Mat dstImg; //dstImg.create(heights, widths, CV_8UC1);
			cvtColor(img, dstImg, CV_BGR2GRAY);


			//yaw 增强 pitch 增强
			for (int j = 0; j < 2; j++)
			{
				if (result_point[0].direct[j] > -theta_offset && result_point[0].direct[j] < theta_offset)
				{
					Rect rect;
					LandMark landmark(result_point[0]);
					bool success = getFaceRect(pd, model_id, dstImg, landmark, rect);
					if (success)
					{
						vector all_rect;
						getoffsetRect(rect, all_rect, img.cols, img.rows, 4);
						for (int i = 0; i < 2; i++)
						{
							LandMark dst_landmark;
							//vector filenames;
							//splitString(landmark.fileName, filenames, "/");
							//string filename = filenames[filenames.size()-1];
							dst_landmark.fileName = text_pre[temp] + to_string(countNum) + ".png";

							//cout << img.rows << " " << rotat_img.cols << " " << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;

							Mat roi_face = img(all_rect[i]);
							cv::resize(roi_face, roi_face, Size(SIZE, SIZE));
							//坐标转换
							for (int k = 0; k < 5; k++)
							{
								dst_landmark.visible[k] = landmark.visible[five_points[k]];
								dst_landmark.points[k].x = ((float)SIZE / all_rect[i].width) * (landmark.points[five_points[k]].x - all_rect[i].x);
								dst_landmark.points[k].y = ((float)SIZE / all_rect[i].height) * (landmark.points[five_points[k]].y - all_rect[i].y);
							}
							imwrite(result_img + dst_landmark.fileName, roi_face);
							rotation_point.push_back(dst_landmark);
							countNum++;

						}
					}

				}
			}
			// roll 增强
			if (roll > -theta_offset && roll < theta_offset)
			{
				for (int i = -1; i < 2; i = i + 2)
				{
					Mat rotat_img;
					LandMark landmark(result_point[0]);
					int theta = (rand() % theta_max + theta_offset) * i;
					rotation(theta, img, rotat_img, Size(SIZE, SIZE), landmark, CURRENT_LANDMARK);

					Mat dstImg; //dstImg.create(heights, widths, CV_8UC1);
					cvtColor(rotat_img, dstImg, CV_BGR2GRAY);

					//for (int j = 0; j < CURRENT_LANDMARK; j++)
					//{
					//	circle(rotat_img, Point(landmark.points[j]), 2, Scalar(255, 0, 0));
					//}
					//imshow("img", rotat_img);
					//waitKey(0);

					LandMark dst_landmark;

					//vector filenames;
					//splitString(landmark.fileName, filenames, "/");
					//string filename = filenames[filenames.size()-1];

					dst_landmark.fileName = text_pre[temp] + to_string(countNum) + ".png";
					Rect rect;
					bool success = getFaceRect(pd, model_id, dstImg, landmark, rect);
					if (success)
					{
						//cout << rotat_img.rows << " " << rotat_img.cols << " " << rect.x << " " << rect.y << " " << rect.width << " " << rect.height << endl;
						Mat roi_face = rotat_img(rect);
						cv::resize(roi_face, roi_face, Size(SIZE, SIZE));
						//坐标转换
						for (int k = 0; k < 5; k++)
						{
							dst_landmark.visible[k] = landmark.visible[five_points[k]];
							dst_landmark.points[k].x = ((float)SIZE / rect.width) * (landmark.points[five_points[k]].x - rect.x);
							dst_landmark.points[k].y = ((float)SIZE / rect.height) * (landmark.points[five_points[k]].y - rect.y);
						}
						imwrite(result_img + dst_landmark.fileName, roi_face);
						rotation_point.push_back(dst_landmark);
						countNum++;
					}
				}
			}
			*/



			//}
		}
		if (rotation_point.size() > 0)
		{
			dpUtil.writePointVisibletoFile(result_txt, rotation_point, NEED_LANDMARK);
			rotation_point.clear();
		}
		system("PAUSE");
		return 0;
}