画saliency intensity map图, caffe

以JAA检测笑脸为例，已经有一个训练好的模型了，现在要查看saliency intensity map。

基本想法是给一张图片，forward之后再backward回来，看data层的diff。

1）deploy.prototxt增加一行force_backward: true，否则caffe不支持backword

2) 先forward再backward，把data的diff保存下来
注意代码中

1. 对au_net.blobs[‘au_probNAU’].diff的赋值, 根据实际图片情况来，比如我这张图片是AU6和AU12，分别是第4和第6个AU，其他AU没有出现
2. diffs = au_net.backward(start=‘au_prob_slice’)[‘data’]，这里代表从’au_prob_slice’层backwards到’data’
3. dif = np.array(au_net.blobs[‘data’].diff).reshape((3,176,176)) , 取出data的diff，即是我们所求

import numpy as np
import matplotlib.pyplot as plt

import os
import sys
import numpy as np
import cv2
from PIL import Image
os.environ["GLOG_minloglevel"] = "2"
sys.path.insert(0,"Algorithms/JAA/caffe/python")
import caffe
def load_image(path):
    img = cv2.imread(path,cv2.IMREAD_COLOR) # BGR
    img = cv2.resize(img,(176,176))
    im = (img-128.0)*0.0078125        
    return im
crop_offset = 12
crop_size = 176
gpu = 0
caffe.set_mode_gpu()
caffe.set_device(gpu)
img_name = "smile.jpg"
#load model
img_path_prefix = "Algorithms-Test/Database/algined_faces/"
au_net_model = "Algorithms-Test/JAA/deploy.prototxt"     
au_net_weights = "Algorithms-Test/JAA/models/2-3-AUNet_iter_160000.caffemodel"
au_net = caffe.Net(au_net_model, au_net_weights, caffe.TEST)
transformer = caffe.io.Transformer({"data": au_net.blobs["data"].data.shape})
transformer.set_transpose("data", (2,0,1))
start = True
au_fc = 0
im = load_image(img_name)       
au_net.blobs["data"].data[0] = transformer.preprocess("data", im)
au_net.forward()
au_probNAU = au_net.blobs["au_probNAU"].data
au_net.blobs['au_probNAU'].diff[0][0][0][0] = 0 - au_probNAU[0][0][0][0]
au_net.blobs['au_probNAU'].diff[0][0][1][0] = 0 - au_probNAU[0][0][1][0]
au_net.blobs['au_probNAU'].diff[0][0][2][0] = 0 - au_probNAU[0][0][2][0]
au_net.blobs['au_probNAU'].diff[0][0][3][0] = 1 - au_probNAU[0][0][3][0] #AU6
au_net.blobs['au_probNAU'].diff[0][0][4][0] = 0 - au_probNAU[0][0][4][0]
au_net.blobs['au_probNAU'].diff[0][0][5][0] = 1 - au_probNAU[0][0][5][0] #AU12
au_net.blobs['au_probNAU'].diff[0][0][6][0] = 0 - au_probNAU[0][0][6][0]
au_net.blobs['au_probNAU'].diff[0][0][7][0] = 0 - au_probNAU[0][0][7][0]
au_net.blobs['au_probNAU'].diff[0][0][8][0] = 0 - au_probNAU[0][0][8][0]
au_net.blobs['au_probNAU'].diff[0][0][9][0] = 0 - au_probNAU[0][0][9][0]
au_net.blobs['au_probNAU'].diff[0][0][10][0] = 0 - au_probNAU[0][0][10][0]
au_net.blobs['au_probNAU'].diff[0][0][11][0] = 0 - au_probNAU[0][0][11][0]
diffs = au_net.backward(start='au_prob_slice')['data']
dif = np.array(au_net.blobs['data'].diff).reshape((3,176,176))
dif = dif - dif.min()
dif = dif/dif.max() 
dif = dif.transpose(1, 2, 0) * 255
np.save(img_name + '_dif_AU12.npy',dif)
im = Image.fromarray(dif.astype(np.uint8))
im.save(img_name + "_dif_AU12.jpg")

3）画图展示
以下分别是saliency intensity map，原图和saliency intensity map覆盖在一起情况


声明：我不是很确定这样做画这个图有没有问题，如果有发现问题欢迎留言或邮件讨论（[email protected]）

不知道作者们是怎么做的，他们的很漂亮：