Ceres计算Bundle Adjustment
开发Ceres的主要原因之一是我们需要解决大规模的Bundle Adjustment，简称BA。
给定一组测量的图像特征和对应关系，Bundle Adjustment的目标是找到使重投影误差最小的3D点位置和相机参数。这个优化问题通常被描述为非线性最小二乘法问题，要最小化的目标函数即为测得图像中特征点位置和相机成像平面上的对应的投影之间的差的平方。Ceres例程使用了BAL数据集 。
第一步是定义一个模板化functor来计算残差，在这个问题中也就是的重投影误差。每个残差值依赖于空间点的位置（3个参数）和相机参数（9个参数）。九个相机参数包含用Rodriques’ axis-angle表示的旋转向量（3个参数），平移参数（3个参数），以及焦距和两个径向畸变参数。相机模型的详细描述也可以在BAL的主页找到。
/****************************************************************************************
* 小孔相机模型.
* camera用9个参数表示， 3 个表示旋转, 3 个表示平移, 1个焦距，2个径向畸变
* 中心点没有建模，假设为图片中心
*****************************************************************************************/
struct SnavelyReprojectionError {
SnavelyReprojectionError(double observed_x, double observed_y):observed_x(observed_x), observed_y(observed_y) {}
template
bool operator()(const T* const camera, const T* const point, T* residuals) const {
// camera[0,1,2] are the angle-axis rotation.
T p[3];
ceres::AngleAxisRotatePoint(camera, point, p);
// camera[3,4,5] are the translation.
p[0] += camera[3];
p[1] += camera[4];
p[2] += camera[5];
// Compute the center of distortion. The sign change comes from
// the camera model that Noah Snavely's Bundler assumes, whereby
// the camera coordinate system has a negative z axis.
T xp = - p[0] / p[2];
T yp = - p[1] / p[2];
// Apply second and fourth order radial distortion.
const T& l1 = camera[7];
const T& l2 = camera[8];
T r2 = xp*xp + yp*yp;
T distortion = 1.0 + r2 * (l1 + l2 * r2);
// Compute final projected point position.
const T& focal = camera[6];
T predicted_x = focal * distortion * xp;
T predicted_y = focal * distortion * yp;
// The error is the difference between the predicted and observed position.
residuals[0] = predicted_x - observed_x;
residuals[1] = predicted_y - observed_y;
return true;
}
// Factory to hide the construction of the CostFunction object from
// the client code.
static ceres::CostFunction* Create(const double observed_x, const double observed_y) {
return (new ceres::AutoDiffCostFunction(
new SnavelyReprojectionError(observed_x, observed_y)));
}
double observed_x;
double observed_y;
}; 注意这个例子相较之前的例子比较复杂，用解析算法求解雅可比行列式非常困难。如果使用自动微分法就会容易很多。函数实现代码如AngleAxisRotatePoint()可以在include/ceres/rotation.h中找到。
给定这个functor，BA问题可以构造如下：
// Create residuals for each observation in the bundle adjustment problem. The
// parameters for cameras and points are added automatically.
ceres::Problem problem;
for (int i = 0; i < bal_problem.num_observations(); ++i) {
// Each Residual block takes a point and a camera as input and outputs a 2
// dimensional residual. Internally, the cost function stores the observed
// image location and compares the reprojection against the observation.
ceres::CostFunction* cost_function = SnavelyReprojectionError::Create(observations[2 * i + 0],
observations[2 * i + 1]);
problem.AddResidualBlock(cost_function,
NULL /* squared loss */,
bal_problem.mutable_camera_for_observation(i),
bal_problem.mutable_point_for_observation(i));
}请注意，BA问题构造与曲线拟合示例非常相似-即为每个观测值增加一个残差块。
由于这是一个大规模的稀疏矩阵（DENSE_QR 无论如何都是大问题），因此解决此问题的一种方法是设置 Solver::Options::linear_solver_type成SPARSE_NORMAL_CHOLESKY，之后调用Solve()。尽管这样做是合理的，但束调整问题具有特殊的稀疏结构，可以利用该稀疏结构更有效地解决它们。Ceres为该任务提供了三个专业的求解器（统称为基于Schur的求解器）。示例代码使用了其中最简单的代码DENSE_SCHUR。
ceres::Solver::Options options;
options.linear_solver_type = ceres::DENSE_SCHUR;
options.minimizer_progress_to_stdout = true;
ceres::Solver::Summary summary;
ceres::Solve(options, &problem, &summary);
std::cout << summary.FullReport() << "\n";完整的程序如下：
#include
#include
#include
#include "ceres/ceres.h"
#include "ceres/rotation.h"
// Read a Bundle Adjustment in the Large dataset.
class BALProblem {
public:
~BALProblem() {
delete[] point_index_;
delete[] camera_index_;
delete[] observations_;
delete[] parameters_;
}
int num_observations() const {
return num_observations_;
}
const double* observations() const {
return observations_;
}
double* mutable_cameras() {
return parameters_;
}
double* mutable_points() {
return parameters_ + 9 * num_cameras_;
}
double* mutable_camera_for_observation(int i) {
return mutable_cameras() + camera_index_[i] * 9;
}
double* mutable_point_for_observation(int i) {
return mutable_points() + point_index_[i] * 3;
}
bool LoadFile(const char* filename) {
FILE* fptr = fopen(filename, "r");
if (fptr == NULL) {
return false;
};
FscanfOrDie(fptr, "%d", &num_cameras_);
FscanfOrDie(fptr, "%d", &num_points_);
FscanfOrDie(fptr, "%d", &num_observations_);
point_index_ = new int[num_observations_];
camera_index_ = new int[num_observations_];
observations_ = new double[2 * num_observations_];
num_parameters_ = 9 * num_cameras_ + 3 * num_points_;
parameters_ = new double[num_parameters_];
for (int i = 0; i < num_observations_; ++i) {
FscanfOrDie(fptr, "%d", camera_index_ + i);
FscanfOrDie(fptr, "%d", point_index_ + i);
for (int j = 0; j < 2; ++j) {
FscanfOrDie(fptr, "%lf", observations_ + 2*i + j);
}
}
for (int i = 0; i < num_parameters_; ++i) {
FscanfOrDie(fptr, "%lf", parameters_ + i);
}
return true;
}
private:
template
void FscanfOrDie(FILE *fptr, const char *format, T *value) {
int num_scanned = fscanf(fptr, format, value);
if (num_scanned != 1) {
LOG(FATAL) << "Invalid UW data file.";
}
}
int num_cameras_;
int num_points_;
int num_observations_;
int num_parameters_;
int* point_index_;
int* camera_index_;
double* observations_;
double* parameters_;
};
/****************************************************************************************
* 小孔相机模型.
* camera用9个参数表示， 3 个表示旋转, 3 个表示平移, 1个焦距，2个径向畸变
* 中心点没有建模，假设为图片中心
*****************************************************************************************/
struct SnavelyReprojectionError {
SnavelyReprojectionError(double observed_x, double observed_y):observed_x(observed_x), observed_y(observed_y) {}
template
bool operator()(const T* const camera, const T* const point, T* residuals) const {
// camera[0,1,2] are the angle-axis rotation.
T p[3];
ceres::AngleAxisRotatePoint(camera, point, p);
// camera[3,4,5] are the translation.
p[0] += camera[3];
p[1] += camera[4];
p[2] += camera[5];
// Compute the center of distortion. The sign change comes from
// the camera model that Noah Snavely's Bundler assumes, whereby
// the camera coordinate system has a negative z axis.
T xp = - p[0] / p[2];
T yp = - p[1] / p[2];
// Apply second and fourth order radial distortion.
const T& l1 = camera[7];
const T& l2 = camera[8];
T r2 = xp*xp + yp*yp;
T distortion = 1.0 + r2 * (l1 + l2 * r2);
// Compute final projected point position.
const T& focal = camera[6];
T predicted_x = focal * distortion * xp;
T predicted_y = focal * distortion * yp;
// The error is the difference between the predicted and observed position.
residuals[0] = predicted_x - observed_x;
residuals[1] = predicted_y - observed_y;
return true;
}
// Factory to hide the construction of the CostFunction object from
// the client code.
static ceres::CostFunction* Create(const double observed_x, const double observed_y) {
return (new ceres::AutoDiffCostFunction(
new SnavelyReprojectionError(observed_x, observed_y)));
}
double observed_x;
double observed_y;
};
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
if (argc != 2) {
std::cerr << "usage: simple_bundle_adjuster \n";
return 1;
}
BALProblem bal_problem;
if (!bal_problem.LoadFile(argv[1])) {
std::cerr << "ERROR: unable to open file " << argv[1] << "\n";
return 1;
}
const double* observations = bal_problem.observations();
// Create residuals for each observation in the bundle adjustment problem. The
// parameters for cameras and points are added automatically.
ceres::Problem problem;
for (int i = 0; i < bal_problem.num_observations(); ++i) {
// Each Residual block takes a point and a camera as input and outputs a 2
// dimensional residual. Internally, the cost function stores the observed
// image location and compares the reprojection against the observation.
ceres::CostFunction* cost_function = SnavelyReprojectionError::Create(observations[2 * i + 0],
observations[2 * i + 1]);
problem.AddResidualBlock(cost_function,
NULL /* squared loss */,
bal_problem.mutable_camera_for_observation(i),
bal_problem.mutable_point_for_observation(i));
}
// Make Ceres automatically detect the bundle structure. Note that the
// standard solver, SPARSE_NORMAL_CHOLESKY, also works fine but it is slower
// for standard bundle adjustment problems.
ceres::Solver::Options options;
options.linear_solver_type = ceres::DENSE_SCHUR;
options.minimizer_progress_to_stdout = true;
ceres::Solver::Summary summary;
ceres::Solve(options, &problem, &summary);
std::cout << summary.FullReport() << "\n";
return 0;
} 针对更加复杂的BA问题，Ceres也提供了更加先进的特征，包括了各种各样的线性解算器、鲁棒的损失函数以及本地参数化。具体内容见代码文件examples/bundle_adjuster.cc。