faster_rcnn c++版本的 caffe 封装（1）

转载请注明出处，楼燚(yì)航的blog，http://www.cnblogs.com/louyihang-loves-baiyan/

由于需要把FasterRCNN做的工程化，因此这里需要对Caffe进行封装。其实封装听起来感觉很高深的样子，其实就是将自己在caffe上再调用的接口做成一个动态库，同时将Caffe的库连着Caffe的那些库依赖一起做成自己工程的库依赖就可以了。如果你只是直接使用Caffe的话，那么到时候直接链接到caffe下面build目录中的libcaffe.so或者libcaffe.a就可以了。如果有对Caffe有C++代码的改动，操作也是一样的，但是如果用了Python 模块比如使用了Python Layer，那么在使用中，还需要为Caffe指定其Python模块的位置。

好了，首先在这里我参照http://blog.csdn.net/xyy19920105/article/details/50440957，感谢他的分享，让我们很受用。首先写成一个C++版本。一开始的版本是这样的，当时只有一个cpp文件。

#include <stdio.h> // for snprintf
#include <string>
#include <vector>
#include <math.h>
#include <fstream>
#include <boost/python.hpp>
#include "caffe/caffe.hpp"
#include "gpu_nms.hpp"
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>
using namespace caffe;
using namespace std;

#define max(a, b) (((a)>(b)) ? (a) :(b))
#define min(a, b) (((a)<(b)) ? (a) :(b))
const int class_num=2;

/*
 * === Class ======================================================================
 * Name: Detector
 * Description: FasterRCNN CXX Detector
 * =====================================================================================
 */
class Detector {
public:
    Detector(const string& model_file, const string& weights_file);
    void Detection(const string& im_name);
    void bbox_transform_inv(const int num, const float* box_deltas, const float* pred_cls, float* boxes, float* pred, int img_height, int img_width);
    void vis_detections(cv::Mat image, int* keep, int num_out, float* sorted_pred_cls, float CONF_THRESH);
    void boxes_sort(int num, const float* pred, float* sorted_pred);
private:
    shared_ptr<Net<float> > net_;
    Detector(){}
};

/*
 * === FUNCTION ======================================================================
 * Name: Detector
 * Description: Load the model file and weights file
 * =====================================================================================
 */
//load modelfile and weights
Detector::Detector(const string& model_file, const string& weights_file)
{
    net_ = shared_ptr<Net<float> >(new Net<float>(model_file, caffe::TEST));
    net_->CopyTrainedLayersFrom(weights_file);
}


//Using for box sort
struct Info
{
    float score;
    const float* head;
};
bool compare(const Info& Info1, const Info& Info2)
{
    return Info1.score > Info2.score;
}

/*
 * === FUNCTION ======================================================================
 * Name: Detect
 * Description: Perform detection operation
 * Warning the max input size should less than 1000*600
 * =====================================================================================
 */
//perform detection operation
//input image max size 1000*600
void Detector::Detection(const string& im_name)
{
    float CONF_THRESH = 0.8;
    float NMS_THRESH = 0.3;
    const int  max_input_side=1000;
    const int  min_input_side=600;

    cv::Mat cv_img = cv::imread(im_name);
    cv::Mat cv_new(cv_img.rows, cv_img.cols, CV_32FC3, cv::Scalar(0,0,0));
    if(cv_img.empty())
    {
        std::cout<<"Can not get the image file !"<<endl;
        return ;
    }
    int max_side = max(cv_img.rows, cv_img.cols);
    int min_side = min(cv_img.rows, cv_img.cols);

    float max_side_scale = float(max_side) / float(max_input_side);
    float min_side_scale = float(min_side) /float( min_input_side);
    float max_scale=max(max_side_scale, min_side_scale);

    float img_scale = 1;

    if(max_scale > 1)
    {
        img_scale = float(1) / max_scale;
    }

    int height = int(cv_img.rows * img_scale);
    int width = int(cv_img.cols * img_scale);
    int num_out;
    cv::Mat cv_resized;

    std::cout<<"imagename "<<im_name<<endl;
    float im_info[3];
    float data_buf[height*width*3];
    float *boxes = NULL;
    float *pred = NULL;
    float *pred_per_class = NULL;
    float *sorted_pred_cls = NULL;
    int *keep = NULL;
    const float* bbox_delt;
    const float* rois;
    const float* pred_cls;
    int num;

    for (int h = 0; h < cv_img.rows; ++h )
    {
        for (int w = 0; w < cv_img.cols; ++w)
        {
            cv_new.at<cv::Vec3f>(cv::Point(w, h))[0] = float(cv_img.at<cv::Vec3b>(cv::Point(w, h))[0])-float(102.9801);
            cv_new.at<cv::Vec3f>(cv::Point(w, h))[1] = float(cv_img.at<cv::Vec3b>(cv::Point(w, h))[1])-float(115.9465);
            cv_new.at<cv::Vec3f>(cv::Point(w, h))[2] = float(cv_img.at<cv::Vec3b>(cv::Point(w, h))[2])-float(122.7717);

        }
    }

    cv::resize(cv_new, cv_resized, cv::Size(width, height));
    im_info[0] = cv_resized.rows;
    im_info[1] = cv_resized.cols;
    im_info[2] = img_scale;

    for (int h = 0; h < height; ++h )
    {
        for (int w = 0; w < width; ++w)
        {
            data_buf[(0*height+h)*width+w] = float(cv_resized.at<cv::Vec3f>(cv::Point(w, h))[0]);
            data_buf[(1*height+h)*width+w] = float(cv_resized.at<cv::Vec3f>(cv::Point(w, h))[1]);
            data_buf[(2*height+h)*width+w] = float(cv_resized.at<cv::Vec3f>(cv::Point(w, h))[2]);
        }
    }

    net_->blob_by_name("data")->Reshape(1, 3, height, width);
    net_->blob_by_name("data")->set_cpu_data(data_buf);
    net_->blob_by_name("im_info")->set_cpu_data(im_info);
    net_->ForwardFrom(0);
    bbox_delt = net_->blob_by_name("bbox_pred")->cpu_data();
    num = net_->blob_by_name("rois")->num();


    rois = net_->blob_by_name("rois")->cpu_data();
    pred_cls = net_->blob_by_name("cls_prob")->cpu_data();
    boxes = new float[num*4];
    pred = new float[num*5*class_num];
    pred_per_class = new float[num*5];
    sorted_pred_cls = new float[num*5];
    keep = new int[num];

    for (int n = 0; n < num; n++)
    {
        for (int c = 0; c < 4; c++)
        {
            boxes[n*4+c] = rois[n*5+c+1] / img_scale;
        }
    }

    bbox_transform_inv(num, bbox_delt, pred_cls, boxes, pred, cv_img.rows, cv_img.cols);
    for (int i = 1; i < class_num; i ++)
    {
        for (int j = 0; j< num; j++)
        {
            for (int k=0; k<5; k++)
                pred_per_class[j*5+k] = pred[(i*num+j)*5+k];
        }
        boxes_sort(num, pred_per_class, sorted_pred_cls);
        _nms(keep, &num_out, sorted_pred_cls, num, 5, NMS_THRESH, 0);
        vis_detections(cv_img, keep, num_out, sorted_pred_cls, CONF_THRESH);
    }

    cv::imwrite("vis.jpg",cv_img);
    delete []boxes;
    delete []pred;
    delete []pred_per_class;
    delete []keep;
    delete []sorted_pred_cls;

}

/*
 * === FUNCTION ======================================================================
 * Name: vis_detections
 * Description: Visuallize the detection result
 * =====================================================================================
 */
void Detector::vis_detections(cv::Mat image, int* keep, int num_out, float* sorted_pred_cls, float CONF_THRESH)
{
    int i=0;
    while(sorted_pred_cls[keep[i]*5+4]>CONF_THRESH && i < num_out)
    {
        if(i>=num_out)
            return;
        cv::rectangle(image,cv::Point(sorted_pred_cls[keep[i]*5+0], sorted_pred_cls[keep[i]*5+1]),cv::Point(sorted_pred_cls[keep[i]*5+2], sorted_pred_cls[keep[i]*5+3]),cv::Scalar(255,0,0));
        i++;
    }
}

/*
 * === FUNCTION ======================================================================
 * Name: boxes_sort
 * Description: Sort the bounding box according score
 * =====================================================================================
 */
void Detector::boxes_sort(const int num, const float* pred, float* sorted_pred)
{
    vector<Info> my;
    Info tmp;
    for (int i = 0; i< num; i++)
    {
        tmp.score = pred[i*5 + 4];
        tmp.head = pred + i*5;
        my.push_back(tmp);
    }
    std::sort(my.begin(), my.end(), compare);
    for (int i=0; i<num; i++)
    {
        for (int j=0; j<5; j++)
            sorted_pred[i*5+j] = my[i].head[j];
    }
}

/*
 * === FUNCTION ======================================================================
 * Name: bbox_transform_inv
 * Description: Compute bounding box regression value
 * =====================================================================================
 */
void Detector::bbox_transform_inv(int num, const float* box_deltas, const float* pred_cls, float* boxes, float* pred, int img_height, int img_width)
{
    float width, height, ctr_x, ctr_y, dx, dy, dw, dh, pred_ctr_x, pred_ctr_y, pred_w, pred_h;
    for(int i=0; i< num; i++)
    {
        width = boxes[i*4+2] - boxes[i*4+0] + 1.0;
        height = boxes[i*4+3] - boxes[i*4+1] + 1.0;
        ctr_x = boxes[i*4+0] + 0.5 * width;
        ctr_y = boxes[i*4+1] + 0.5 * height;
        for (int j=0; j< class_num; j++)
        {

            dx = box_deltas[(i*class_num+j)*4+0];
            dy = box_deltas[(i*class_num+j)*4+1];
            dw = box_deltas[(i*class_num+j)*4+2];
            dh = box_deltas[(i*class_num+j)*4+3];
            pred_ctr_x = ctr_x + width*dx;
            pred_ctr_y = ctr_y + height*dy;
            pred_w = width * exp(dw);
            pred_h = height * exp(dh);
            pred[(j*num+i)*5+0] = max(min(pred_ctr_x - 0.5* pred_w, img_width -1), 0);
            pred[(j*num+i)*5+1] = max(min(pred_ctr_y - 0.5* pred_h, img_height -1), 0);
            pred[(j*num+i)*5+2] = max(min(pred_ctr_x + 0.5* pred_w, img_width -1), 0);
            pred[(j*num+i)*5+3] = max(min(pred_ctr_y + 0.5* pred_h, img_height -1), 0);
            pred[(j*num+i)*5+4] = pred_cls[i*class_num+j];
        }
    }

}

int main()
{
    string model_file = "/home/lyh1/workspace/py-faster-rcnn/models/pascal_voc/VGG_CNN_M_1024/faster_rcnn_alt_opt/faster_rcnn_test.pt";
    string weights_file = "/home/lyh1/workspace/py-faster-rcnn/output/default/yuanzhang_car/vgg_cnn_m_1024_fast_rcnn_stage2_iter_40000.caffemodel";
    int GPUID=0;
    Caffe::SetDevice(GPUID);
    Caffe::set_mode(Caffe::GPU);
    Detector det = Detector(model_file, weights_file);
    det.Detection("/home/lyh1/workspace/py-faster-rcnn/data/demo/car.jpg");
return 0;
}

这个文件对应的CMakeLists.txt，其中对编译部分加了相应的注释。此处需要添加$PYTHONPATH，原因是因为我们在使用中是C++接口调用的Caffe，但是在Caffe运行中调用了Python，因此需要告诉Caffe我们自己的Python模块的路径，比如这里去模型的定义文件中看，里面用了py-faster-rcnn根目录下的lib中的rpn文件夹下的proposal模块，因此需要在$PYTHONPATH中加入这个模块路径...\py-faster-rcnn\lib以及Caffe的Python接口路径...\py-faster-rcnn\caffe-fast-rcnn\python，下面是我在~.bashrc中添加$PYTHONPATH，可供参考。添加之后执行source ~/.bashrc，对bash立即生效，提醒一下.bashrc是在你启动一个bash的时候会被立即执行的。要么你直接在当前bash中直接source，否则需要再打开一个新的bash。还有一点，顺带提一下，加入到$PYTHONPATH之后，如果你系统中有多个caffe，那么你得想一下怎么防止冲突，当时我就给自己埋了个坑，别的caffe调用PYTHON接口时，就直接调到这个加入到$PYTHONPATH中的这个caffe了，所以各种报错，如果可以话可以考虑用Docker来处理这种东西。

如图添加

还是接下来讲一下这个CMakeList.txt。我也是第一次用，之前用Autotools，真是各种麻烦，对Linux下的手动编译都有一些阴影。用Cmake真的是比较傻瓜式的，能把东西梳理的比较清晰。

#This part is used for compile faster_rcnn_demo.cpp
#这里是版本要求，根据自己项目而定，我用了默认的
cmake_minimum_required (VERSION 2.8)
#我们的工程的名字
project (faster_rcnn_demo)

#添加我们要生成的可执行文件的名字，以及相应的源码文件
add_executable(faster_rcnn_demo faster_rcnn_demo.cpp)
#这里添加这个faster_rcnn_demo.cpp所依赖的头文件路径
#首先是Caffe目录的include
#其次是用了gpu_nms.cu，所以也要添加相应的头文件gpu_nms.hpp在py-faster-rcnn根目录下的lib/nms中
#下面就是几个Caffe的依赖项，包括Python
#值得注意的是boost/python。hpp的头文件路径也要加入
#还有opencv的路径，cuda路径，线性代数库路径相应的都要添加
include_directories ( "${PROJECT_SOURCE_DIR}/../caffe-fast-rcnn/include"
    "${PROJECT_SOURCE_DIR}/../lib/nms" 
    /share/apps/local/include
    /usr/local/include 
    /opt/python/include/python2.7
    /share/apps/opt/intel/mkl/include 
    /usr/local/cuda/include )
#在这里值得说一下，target_link_libraries 语句中 生成的目标是可执行文件 后面紧跟的得是动态库的完整路径，否则会出错
#我一开始用的是Link_directoreis然后在后面直接加入了动态库的路径，结果他一直报错，提示找不到库，ORZ真是跪了，
#所以要么在这里直接加入完整路径或者同通过另一条语句find_library()，这种方式也比较好，直接去指定路径查找，返回相应的绝对路径也可避免直接添加地址的问题
#gpu_nms.so 在py-faster-rcnn根目录下的lib\nms中，直接make就会生成这个so文件
target_link_libraries(faster_rcnn_demo /home/lyh1/workspace/py-faster-rcnn/caffe-fast-rcnn/build/lib/libcaffe.so
    /home/lyh1/workspace/py-faster-rcnn/lib/nms/gpu_nms.so 
    /share/apps/local/lib/libopencv_highgui.so 
    /share/apps/local/lib/libopencv_core.so 
    /share/apps/local/lib/libopencv_imgproc.so 
    /share/apps/local/lib/libopencv_imgcodecs.so
    /share/apps/local/lib/libglog.so
    /share/apps/local/lib/libboost_system.so
    /share/apps/local/lib/libboost_python.so
    /share/apps/local/lib/libglog.so
    /opt/rh/python27/root/usr/lib64/libpython2.7.so
    )

编译的时候比较坑爹，还会到这个问题

但是我用find命令查找了一下整个caffe的工程目录下居然没有这个caffe.pb.h 后来Google一下之后才知道需要手动生成，解决办法如下用protoc命令手动生成，并放到include文件夹下

protoc src/caffe/proto/caffe.proto --cpp_out=.
mkdir include/caffe/proto
mv src/caffe/proto/caffe.pb.h include/caffe/proto

最后cmake .然后make编译成功，运行正常。
接下来就是这么打包成动态库了，具体操作在下一个博文中