Opencv 实现屏幕内容提取 JAVA

https://blog.csdn.net/qq_43701766/article/details/105877653

基于Android端开发的，opencv3.4.1。
代码有点时间了细节记得不太清楚。
为了方便已把Logcat内容和保存图片部分删掉了测试的时候自行改动就好。
问题
需要提取图片中矩形显示屏的内容，由此可以想到，提取四个角的坐标再利用投影变换得到显示屏的内容。

裁剪后的图片大概就是这样子
这是在mat里保存的图片所以颜色不一样。

思路
首先显示屏的亮度一般情况会高于周围环境，利用伽马矫正的方法对图像进行调整，便于寻找轮廓采集角点坐标。
原图保留用于裁剪图片。

public static Mat GaMa(Mat src){
    Mat srcClone = src.clone();
    srcClone.convertTo(srcClone, CvType.CV_32F);
    // 0.9 调低亮度 易于提取显示屏轮廓
    Core.pow(srcClone, 0.9, srcClone);
    Core.convertScaleAbs(srcClone, srcClone);
    return srcClone;
}

主要实现方法
进行灰度、二值等处理，得到轮廓与角点。
得到轮廓的时候采用的多边拟合。

public static Mat Cut_Out_TFT(Bitmap img) {

    Mat greyPic = new Mat();
    Mat binPic = new Mat();
    Mat cannyPic = new Mat();
    Mat hierarchy = new Mat();
    Mat srcPic = new Mat();
    Utils.bitmapToMat(img, srcPic);
     
    Mat mRgba = GaMa(srcPic);   // 伽马处理
    
    Imgproc.cvtColor(mRgba, greyPic, Imgproc.COLOR_BGR2GRAY);   //灰度化
    Imgproc.threshold(greyPic, binPic, 80, 255, Imgproc.THRESH_BINARY);
    Imgproc.Canny(binPic, cannyPic, 200, 2.5);  // 边缘检测
 
    List<MatOfPoint> contours = new ArrayList<>();
    Imgproc.findContours(cannyPic, contours, hierarchy, Imgproc.RETR_CCOMP, Imgproc.CHAIN_APPROX_SIMPLE);
    Mat linePic = Mat.zeros(cannyPic.rows(), cannyPic.cols(), CvType.CV_8UC3);
    int index = 0;
    List<MatOfPoint> polyContours = new ArrayList<>();  // 存取轮廓
    for (MatOfPoint point : contours){
    
        MatOfPoint2f curvesPoint = new MatOfPoint2f(point.toArray());//将MatOfPoint转为MatOfPoint2f
        MatOfPoint2f approxCurvesPoint = new MatOfPoint2f();
        Imgproc.approxPolyDP(curvesPoint, approxCurvesPoint, 8, true);//拟合
        MatOfPoint newPoint = new MatOfPoint(approxCurvesPoint.toArray());//将MatOfPoint2f转为MatOfPoint
        polyContours.add(newPoint);
        Imgproc.drawContours(linePic, contours, index, new Scalar(0,255,0), 1, 8, hierarchy);

        index++;
    }
     
    int i = 0;
    int maxArea = 0;
    MatOfPoint2f MaxP = new MatOfPoint2f();
    for (MatOfPoint point : polyContours) {     // 找到单个轮廓中面积最大的
        if (i == 0 || Imgproc.contourArea(point) > Imgproc.contourArea(MaxP)){
            MaxP = new MatOfPoint2f(point.toArray());
            maxArea = i;
        }
        i++；
    }
    Mat polyPic = Mat.zeros(cannyPic.rows(), cannyPic.cols(), CvType.CV_8UC3);
    Imgproc.drawContours(polyPic, polyContours, maxArea, new Scalar(0,255,0), 2);
    return CutPic(srcPic, polyPic); // 裁剪图片方法
}

最终得到的轮廓与角点

裁剪处理方法
根据得到轮廓的角点进行处理然后通过投影变换提取
由于我只需要用到矩形，所以只做了矩形的，不同需求的可以根据代码进行改动。

public static Mat CutPic(Mat srcPic, Mat line){

    // 获取轮廓角点坐标
    Point[] LinePoints = findp(line);
    LinePoints = checkPoint(LinePoints);

    LinePoints = pointSort(LinePoints); // 需裁剪的角点排序
    // 定义裁剪图片的坐标点  左上 右上 左下 右下
    Point[] srcPoints = new Point[4];
    srcPoints[0] = new Point(0, 0);
    srcPoints[1] = new Point(srcPic.cols(), 0);
    srcPoints[2] = new Point(srcPic.cols(), srcPic.rows());
    srcPoints[3] = new Point(0, srcPic.rows());

    // 将点集存储到 List 转换为 Mat
    List<Point> srcP = Arrays.asList(srcPoints[0], srcPoints[1], srcPoints[2], srcPoints[3]);
    List<Point> LineP = Arrays.asList(LinePoints[0], LinePoints[1], LinePoints[2], LinePoints[3]);
    Mat srcM = Converters.vector_Point_to_Mat(srcP, CvType.CV_32F);
    Mat LineM = Converters.vector_Point_to_Mat(LineP, CvType.CV_32F);

    // 透视变换
    Mat outPic = new Mat();
    Mat transMat = Imgproc.getPerspectiveTransform(srcM, LineM); // 得到变换矩阵
    Imgproc.warpPerspective(srcPic, outPic, transMat, srcPic.size(), Imgproc.INTER_LINEAR + Imgproc.WARP_INVERSE_MAP,
            1,new Scalar(0)); // 得到裁剪后图像

    return outPic;
}

下面是用到的方法：

findp方法：
得到角点

public Point[] findp(Mat mat) {
        
        final double qualityLevel = 0.01, k = 0.04;
        
        MatOfPoint corners = new MatOfPoint();
        Imgproc.cvtColor(mat, mat, Imgproc.COLOR_BGR2GRAY);//灰度化
        Imgproc.goodFeaturesToTrack(mat, corners, 50, qualityLevel, 10,
                new Mat(), 3, true, k);

        Point[] pCorners = corners.toArray();
        return pCorners;
    }

checkPoint方法：
减少角点数量

public Point[] checkPoint(Point[] points) {
    int lastLength = -1;
    int thisLength = 0;
    Point[] lp = points;
    Point[] np;
    while (true) {
        np = checkPointOnce(lp);
        thisLength = np.length;
        if (thisLength == lastLength) {
            break;
        }
        lastLength = thisLength;
        lp = np;
    }
    return np;
}

private Point[] checkPointOnce(Point[] points) {
        int length = points.length;
        boolean flag = false;// 是否找到可删除点
        if (length < 4) {
            return points;// 如果小于四个点 免了判断
        }
        label: for (int i = 0; i < length; i++) {// 得到点1
            for (int j = 0; j < length; j++) {// 得到点2
                if (j == i) {
                    continue;
                }
                for (int k = 0; k < length; k++) {// 得到点3
                    if (k == j || k == i) {
                        continue;
                    }
                    // int slope = 0;//斜率
                    double d1 = twoPointsAngel(points[i], points[j]);// i,j直线角度
                    double d2 = twoPointsAngel(points[i], points[k]);// i,k直线角度
                    double angelMin = d1 - d2;
                    if (Math.abs(angelMin) < 10) {// 如果倾角非常接近，删除中间的点
                        int needDelete = deleteMiddlePointToNull(points[i],
                                points[j], points[k]);
                        if (needDelete == 1) {
                            points[i] = null;
                        } else if (needDelete == 2) {
                            points[j] = null;
                        } else if (needDelete == 3) {
                            points[k] = null;
                        }
                        flag = true;
                        break label;
                    }
                }
            }
        }
        if (flag) {
            Point[] newPoints = new Point[length - 1];
            int index = 0;
            for (Point p : points) {// 准备一个没有空值的新数组
                if (null != p) {
                    newPoints[index] = p;
                    index++;
                }
            }
            return newPoints;
        } else {
            return points;
        }
    }
    /**
     * 删除三点中处于中间的点
     */
    private int deleteMiddlePointToNull(Point p1, Point p2, Point p3) {
        double a = p1.x + p1.y;
        double b = p2.x + p2.y;
        double c = p3.x + p3.y;
        if ((a > b && b > c) || (a < b && b < c)) {// b在中间
            return 2;
        } else if ((c > a && a > b) || (c < a && a < b)) {// a在中间
            return 1;
        } else {
            return 3;
        }
    }

pointSort方法
投影变换需要角点顺序一致
故将显示屏的四个角点进行排序
左上 右上 右下 左下

public static Point[] pointSort(Point[] LinePoints){

    // 坐标排序 左上 右上 右下 左下
    Point[] a = new Point[4];
    double xmin = LinePoints[0].x, xmax = 0, ymin = LinePoints[0].y, ymax = 0;
    for (int i = 0; i < LinePoints.length; i++){
        if (LinePoints[i].x > xmax) xmax = LinePoints[i].x;
        if (LinePoints[i].x < xmin) xmin = LinePoints[i].x;
        if (LinePoints[i].y > ymax) ymax = LinePoints[i].y;
        if (LinePoints[i].y < ymin) ymin = LinePoints[i].y;
    }
    a[0] = new Point(xmin, ymin);
    a[1] = new Point(xmax, ymin);
    a[2] = new Point(xmax, ymax);
    a[3] = new Point(xmin, ymax);
    return a;
}

基本上就是这样，具体的自行测试，当时做这个用的时间比较短印象不太深，太具体的也记不清了，有问题提出来，希望对大家有所帮助。