Android之基于Facenet模型比对视频中的人脸

前言

继续前面 MTCNN移植安卓并检测视频中人脸 ，已经检测到人脸，那么需要对所检测的人脸和本地的人脸数据做比对，此时采用的是基于Facenet模型，它的逻辑和实现原理 可以看
之前一篇文章是通过python介绍，访问 基于facenet做人脸比对

在这里也免费发布了一个chat 希望朋友能点击一下，免费的噢！
https://gitbook.cn/gitchat/activity/5c2050fb1c648b470dce1615

介绍

下面是将Facenet移植到Android上来使用

• 1、首先新建一个Facenet类

package com.cayden.face.facenet;

import android.content.res.AssetManager;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.os.Environment;
import android.util.Log;

import org.tensorflow.contrib.android.TensorFlowInferenceInterface;

import java.io.File;
import java.io.FileInputStream;

/*
import android.graphics.Bitmap;
import android.os.Trace;
import android.util.Log;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.PriorityQueue;
import java.util.Vector;
*/

/**功能：人脸转换为512维特征向量
 */
public class Facenet{
    private static final String MODEL_FILE  = "file:///android_asset/20180402-114759.pb";
    private static final String INPUT_NAME  = "input:0";
    private static final String OUTPUT_NAME = "embeddings:0";
    private static final String PHASE_NAME  = "phase_train:0";
    private static final String[] outputNames = new String[] {OUTPUT_NAME};
    //输入图片大小.(图片非此大小，会rescale)
    private static final int INPUT_SIZE=160;
    private float[] floatValues;  //保存input的值
    private int[] intValues;      //像素值
    private AssetManager assetManager;
    private TensorFlowInferenceInterface inferenceInterface;

    private static class SingletonInstance {
        private static final Facenet INSTANCE = new Facenet();
    }

    public static Facenet getInstance() {
        return SingletonInstance.INSTANCE;
    }

    private Facenet() {

        loadModel();
        floatValues=new float[INPUT_SIZE*INPUT_SIZE*3];
        intValues = new int[INPUT_SIZE * INPUT_SIZE];
    }



    private boolean loadModel(){
        //AssetManager
        try {
            String file= Environment.getExternalStorageDirectory().getAbsolutePath()+"/20180402-114759.pb";

            FileInputStream fileInputStream=new FileInputStream(new File(file));
            inferenceInterface = new TensorFlowInferenceInterface(fileInputStream);
            Log.d("Facenet","[*]load model success");
            fileInputStream.close();
        }catch(Exception e){
            Log.e("Facenet","[*]load model failed"+e);
            return false;
        }
        return true;
    }
    //Bitmap to floatValues
    private int normalizeImage(final Bitmap _bitmap){
        // (0) bitmap缩放到INPUT_SIZE*INPUT_SIZE
        float scale_width=((float)INPUT_SIZE)/_bitmap.getWidth();
        float scale_height=((float)INPUT_SIZE)/_bitmap.getHeight();
        Matrix matrix = new Matrix();
        matrix.postScale(scale_width,scale_height);
        Bitmap bitmap = Bitmap.createBitmap(_bitmap,0,0,_bitmap.getWidth(),_bitmap.getHeight(),matrix,true);
        //Log.d("Facenet","[*]bitmap size:"+bitmap.getHeight()+"x"+bitmap.getWidth());
        // (1) 将像素映射到[-1,1]区间内
        float imageMean=127.5f;
        float imageStd=128;
        bitmap.getPixels(intValues,0,bitmap.getWidth(),0,0,bitmap.getWidth(),bitmap.getHeight());
        for (int i=0;i> 16) & 0xFF) - imageMean) / imageStd;
            floatValues[i * 3 + 1] = (((val >> 8) & 0xFF) - imageMean) / imageStd;
            floatValues[i * 3 + 2] = ((val & 0xFF) - imageMean) / imageStd;
        }
        //Log.d("Facenet","[*]normalizeImage");
        //Log.d("Facenet","[*]normalizeImage"+intValues.length);
        return 0;
    }
    public FaceFeature recognizeImage(final Bitmap bitmap){
        //Log.d("Facenet","[*]recognizeImage");
        //(0)图片预处理，normailize
        normalizeImage(bitmap);
        //(1)Feed
        try {
            inferenceInterface.feed(INPUT_NAME, floatValues, 1, INPUT_SIZE, INPUT_SIZE, 3);
            boolean [] phase=new boolean[1];
//            phase[0]=false;
            inferenceInterface.feed(PHASE_NAME,phase);

        }catch (Exception e){
            Log.e("Facenet","[*] feed Error\n"+e);
            return null;
        }
        //(2)run
       // Log.d("Facenet","[*]Feed:"+INPUT_NAME);
        try {
            inferenceInterface.run(outputNames, false);
        }catch (Exception e){
            Log.e("Facenet","[*] run error\n"+e);
            return null;
        }
        //(3)fetch
        FaceFeature faceFeature=new FaceFeature();
        float[] outputs=faceFeature.getFeature();
        try {
            inferenceInterface.fetch(OUTPUT_NAME, outputs);
        }catch (Exception e){
            Log.e("Facenet","[*] fetch error\n"+e);
            return null;
        }
        return faceFeature;
    }
}

• 2、计算特征向量的欧式距离
  人脸特征(512维特征值)

package com.cayden.face.facenet;

/**
 * Created by caydencui on 2018/9/6.
 * 人脸特征(512维特征值)
 * 相似度取特征向量之间的欧式距离.
 */
public class FaceFeature {
    public static final int DIMS=512;
    private float fea[];
    public FaceFeature(){
        fea=new float[DIMS];
    }
    public float[] getFeature(){
        return fea;
    }

    public void setFea(float[] fea) {
        this.fea = fea;
    }

    //比较当前特征和另一个特征之间的相似度
    public double compare(FaceFeature ff){
        double dist=0;
        for (int i=0;i

• 3、对检测的人脸进行特征值提取 并和已存在本地的特征值做比对

 mNxpRtsp = new Rtsp(this, new RtspCallbackInterface() {
            @Override
            public void decodeOutputBuffer(int frameLen, byte[] bytes, long width, long height) {
                Log.d(TAG, "Get frameLen :" + frameLen + " width :" + width + " height :" + height);
                if (frameLen == 0) return;
                if (iw == 0) {
                    iw = (int) width;
                    ih = (int) height;

                    int surface_w = mVideoTexture.getWidth();
                    int surface_h = mVideoTexture.getHeight();
                    scale_bit = (float) surface_h / ih;
                    ViewGroup.LayoutParams params = draw_view.getLayoutParams();
                    params.width = surface_w;
                    params.height = surface_w;
                    DLog.d("scale_bit:" + scale_bit + ",surface_w:" + surface_w + ",surface_h:" + surface_h + ",iw:" + iw + ",ih:" + ih);
                    draw_view.requestLayout();
                }

//                if (isSave) {
//                    mPictureSave(bytes);
//                    isSave = false;
//                }
                /**
                 * 将bytes转为bitmap
                 */
                synchronized (lock) {
                    Bitmap bitmap = null;
                    byte[] NV21 = new byte[bytes.length];
                    NV12ToNV21(bytes, NV21, iw, ih);
                    NV21ToBitmap nv21ToBitmap = new NV21ToBitmap(MainActivity.this);
                    bitmap = nv21ToBitmap.nv21ToBitmap(NV21, iw, ih);
                    if (isSave) {//保存图片
                        ImageUtils.saveImg(bitmap);
                        isSave = false;
                    }
                    Vector boxes = mtcnn.detectFaces(bitmap, 20);

                    drawAnim(boxes, draw_view, scale_bit, 1, "");

                    if (boxes.size()==0) return ;
                    for (int i=0;i mResgist =FaceDB.getInstance().mRegister;
                    double tempScore=-1;
                    for(int i=0;itemp){
                                foundName=faceRegist.mName;
                                tempScore=temp;
                            }
                        }
                        Log.d(TAG,">>>>>>>>>>temp="+temp+",tempScore="+tempScore+",foundName："+foundName);
                    }
                    cmp=tempScore;
                    runOnUiThread(new Runnable() {
                        @Override
                        public void run() {

                            tv_result.setText(String.format("名字:%s  相似度 :  %.4f", foundName,cmp) );
                        }
                    });
                }

            }
        });

• 4、比对结果如图所示
  

这里的伐值可以设置为0.8 ，但是官网给出的是1.1 ，可以理解为低于伐值表示是同一个人！

项目源码

https://github.com/cayden/facesample
本项目主要基于vlc来播放流媒体视频
主要包含以下内容

• 1、使用已经编译的libvlc来播放流媒体视频
• 2、使用MTCNN进行人脸识别并标记人脸
• 3、保存标记的人脸图片
• 4、使用FACENET进行人脸比对
• 未完待续…

v1.0.3

• 1, 通过FACENET获取脸部特征数据
• 2, 人脸比对，找出相似度最高的人

v1.0.2

• 1, 通过MTCNN检测人脸
• 2, 对人脸进行标记

v1.0.1

• 1, 获取返回的视频流数据
• 2, 将数据nv12转换为nv21,并保存图片

v1.0.0

• 1, added libvlc
• 2, support for playing rtsp video stream

感谢大家的阅读，也希望能转发并关注我的公众号