1.编写代码，模拟定时对流数据进行定时刷新

package com.wutos.sync.component;

import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.Filters;
import com.wutos.sync.domain.dto.DataMogodbDTO;
import com.wutos.sync.domain.dto.DataSyncDTO;
import com.wutos.sync.domain.dto.DiskStatusMogodbDTO;
import com.wutos.sync.domain.entity.DataSyncLog;
import com.wutos.sync.enums.SyncDataSource;
import com.wutos.sync.enums.SyncOptType;
import com.wutos.sync.mq.ActiveMqManage;
import com.wutos.sync.service.IDataSyncService;
import com.wutos.sync.utils.UUIDUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.data.mongodb.core.MongoTemplate;
import javax.jms.JMSException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;

public class MogodbTask implements Runnable {

    Logger logger = LoggerFactory.getLogger(MogodbTask.class);
    /**
     * 时间节点结束节点
     */
    private long end;
    /**
     * 时间节点开始节点
     */
    private long nextstart;

    private MogodbTask mogodbTask;
    /**
     * 定时的时间周期,每隔5个小时刷新一次
     */
    static final long timeClock = 60 * 60* 5;

    private StringBuffer stringBuffer = new StringBuffer();

    /**
     * 是否成功的标志位
     */
    public static Boolean successFlag = null;

    public static AtomicLong size = new AtomicLong(0L);//原子性线程操作

    private long num = 1;

    private long CurrentThread = 0;//当前线程数

    private CountDownLatch over;//监控线程数

    private boolean isRunable = true;

    private String platformId;  //

    private IDataSyncService dataSyncService;

    private ActiveMqManage activeMqManage;

    MongoTemplate oplogMongoTemplate;

    long timestrap_value ;

    public static String str = null; // 将StringBuffer转化成字符串
    public static StringBuffer sb = new StringBuffer(); // StringBuffer便于字符串的增删改查操作

    /**
     * Mogodb监听查询间隔：毫秒
     */

    public MogodbTask(long start, long end, String platformId) {
        this.end = end;
        this.nextstart = start;
        this.platformId = platformId;
        this.dataSyncService = (IDataSyncService) SpringUtil.getBean("dataSyncService");
        this.activeMqManage = (ActiveMqManage) SpringUtil.getBean("dataSyncMQ");
        this.oplogMongoTemplate = (MongoTemplate) SpringUtil.getBean("oplogMongoTemplate");
    }

    /**
     * 使用锁机制实现定期的去读取并保证线程安全，mogodb的oplog是不支持批量更新的
     */
    synchronized private long getStart() {
        long temp = nextstart;//
        nextstart = nextstart + timeClock;//时间为一分钟
        return temp;
    }


    synchronized private long getNum() {
        return num++;
    }

    @Override
    public void run() {
        logger.info("hsss ----------------------------------!");
        mogodbdata();
    }


    public void stop() {
        this.isRunable = false;
        logger.info("Mogodbtask is stop!");
    }

    /**
     * 删除从oplog中取出来的日志变化
     * { "ts" : { "$timestamp" : { "t" : 1544078644, "i" : 1 } }, "t" : { "$numberLong" : "1" },
     * "h" : { "$numberLong" : "6632986490438552198" }, "v" : 2, "op" : "d", "ns" : "db_dataCollection.t_diskStatus",
     * "ui" : { "$binary" : "AkPf/x94/kVKWhNPSbN8hQ==", "$type" : "03" }, "wall" : { "$date" : 1544078644457 }, "o" : { "_id" : "14" } }
     *
     * 更新从oplog中取出来的日志变化
     * { "ts" : { "$timestamp" : { "t" : 1544078277, "i" : 1 } }, "t" : { "$numberLong" : "1" },
     * "h" : { "$numberLong" : "-338099272864304700" }, "v" : 2, "op" : "u", "ns" : "db_dataCollection.t_diskStatus",
     * "ui" : { "$binary" : "AkPf/x94/kVKWhNPSbN8hQ==", "$type" : "03" }, "o2" : { "_id" : "13" },
     * "wall" : { "$date" : 1544078277111 }, "o" : { "$v" : 1, "$set" : { "modifiedOn" : "2018-12-05 16:49:45" } } }
     *
     * 插入从oplog中取出来的变化
     * { "ts" : { "$timestamp" : { "t" : 1543999782, "i" : 1 } }, "t" : { "$numberLong" : "1" },
     * "h" : { "$numberLong" : "1559286620930196252" }, "v" : 2, "op" : "i",
     * "ns" : "db_dataCollection.t_cameraChannel", "ui" : { "$binary" : "HUuFNtANi+2TUgHeIvIovg==", "$type" : "03" },
     * "wall" : { "$date" : 1543999782209 }, "o" : { "_id" : "0", "blurStatus" : 0, "brightnessStatus" : 0,
     * "cameraName" : "一楼", "cameraSn" : "10001", "cameraStatus" : 0, "contrastStatus" : 0, "coverStatus" : 0,
     * "deviceId" : "14", "ditherStatus" : 0, "frozenStatus" : 0, "lossStatus" : 0, "modifiedOn" : "2018-12-05 16:49:42",
     * "noiseStatus" : 0, "scenechangeStatus" : 0, "striationStatus" : 0, "unbalanceStatus" : 0 } }
     *
     *
     */

    private void mogodbdata() {
        MongoTemplate oplogMongoTemplate = (MongoTemplate) SpringUtil.getBean("oplogMongoTemplate");
        MongoCollection oplogRs = oplogMongoTemplate.getCollection("oplog.rs");

        List tableNames = new ArrayList<>();
        //对于应该去进行监控的表添加到响应的tableNames中去。
        //tableNames.add("db_dataCollection.t_deviceData");
        tableNames.add("db_dataCollection.t_diskStatus");//7个键值对
        tableNames.add("db_dataCollection.t_cameraChannel");//16个键值对
        /**
         * mogodb的游标获取，以及得到轮询的所有的值:
         */ //左右每次处理的条数为100
        MongoCursor cursor = oplogRs.find(Filters.in("ns", tableNames)).iterator();
        String timestrap=null;
        //TODO 这块用来取时间戳作为处理数据流的依据
        while (cursor.hasNext()) {
            Document doc1 = cursor.next();
            JSONObject jsontime = JSONObject.parseObject(doc1.toJson());
            System.out.println("--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------");
            String timelist = jsontime.getString("ts");
            JSONObject datalist = JSONObject.parseObject(timelist);
            //作为从oplog取出的一个值:
            String temp_strap = datalist.getString("$timestamp");
            JSONObject tlist = JSONObject.parseObject(temp_strap);
            timestrap = tlist.getString("t");
            long flag=Long.parseLong(timestrap);
           // TODO 得出时间戳的临界值
            sb.append(flag);//将字符串追加到
            sb.append("%");
            }
            str=sb.toString();//将StringBuffer类型数据转换成String类型
            String[] params = StringUtils.split(str, "%");


            //TODO 获取当前事件毫秒数,并转换为Unix的整点时间戳
            Long currentTimestamps=System.currentTimeMillis();
            Long oneDayTimestamps= Long.valueOf(60*60*24*1000);
            //TODO 每天时间的0点时间戳,并转换为UNIX时间戳的形式
            long current_hour=(currentTimestamps-(currentTimestamps+60*60*8*1000)%oneDayTimestamps)/1000;
            for(int i=0;i<24;i++){
                current_hour=current_hour+timeClock;
                long hour_time=System.currentTimeMillis()/1000-timeClock;
                while(current_hour>hour_time){
                    //TODO 获取当前时间整点时间戳为current_hour，作为一个临界点进行处理,
                    //TODO 当前时间整点数加当前之后的的一个小时,时间戳.
                    long current_end=current_hour+timeClock;
                                    //筛选满足条件的timestrap
                                    for(int j=0;jcurrent_hour&×trap_value syncList = Lists.newArrayList();
        String tableName = object.getString("ns");//表名称
        String eventType = object.getString("o"); //更改的事件名称,具体的事件
        String pk_id = object.getString("o2");
        JSONObject  hss= JSONObject.parseObject(pk_id);
        DataSyncLog log = new DataSyncLog();
        /**
         * 将主键Id插入到pk_Id中去
         */
        log.setPkId(hss.getString("_id"));
        System.out.println("输出主键_id的值"+hss.getString("_id"));
        log.setDataSource(SyncDataSource.MONGODB.getValue());//2//绑定数据源
        log.setPlatformId(platformId);//初始化方法从service中取值取出来
        log.setId(UUIDUtils.getUuidCode());//uuid
        log.setCreatedOn(LocalDateTime.now());//新增时间
        log.setModifiedOn(log.getCreatedOn());//更改时间
        log.setTableName(tableName);//将表名写入
        if (object.getString("op").matches("i")) {
            System.out.println("进入插入数据封装");
            log.setOperateType(SyncOptType.INSERT.getValue());//i=1
            parseColumn(eventType, log);
        } else if (object.getString("op").matches("u")) {
            System.out.println("进入更改数据封装");
            log.setOperateType(SyncOptType.UPDATE.getValue());//2
            parseColumn(eventType, log);
        } else if (object.getString("op") == "d") {
            System.out.println("进入删除数据封装");
            log.setOperateType(SyncOptType.DELETE.getValue());//3
            parseColumn(eventType, log);
        }
        syncList.add(log);
        if (CollectionUtils.isNotEmpty(syncList)) {
            dataSyncService.insertList(syncList);
        }
        // TODO 放入消息队列并调API
        DataSyncDTO dto = new DataSyncDTO();
        dto.setDataSource(SyncDataSource.MONGODB.getValue());
        dto.setDataSyncLogList(syncList);
        //
        try {
            System.out.println("准备进入hss写的mogodb端的mQ");
            activeMqManage.sendMessage("queue.DataSync", dto);
        } catch (JMSException e) {
            e.printStackTrace();
        }
    }


    /**
     * 将mogodb中的对象转成自己的数据传输对象，进行json转换,并且封装到log中去。
     *
     * @param
     * @param log
     */
    private void parseColumn(String eventType, DataSyncLog log) {
        System.out.println("进入真实有价值的数据的拆开以及封装,查看一下原来的值" + eventType);
        //List resultList = Lists.newArrayList();
        JSONObject jsStr = JSONObject.parseObject(eventType); //将字符串{“id”：1}
        //获取主键id的判定条件如下所示
        System.out.println("输出jsStr_id"+jsStr.get("_id"));
            logger.info("update mogodb data");
            String  data = jsStr.getString("$set");
            System.out.println("输出firstkey的值"+data);
            JSONObject datalist = JSONObject.parseObject(data);
            System.out.println("输出数据库中表字段的集合"+datalist);
            System.out.println("datalist.getString(\"_id\")"+datalist.getString("_id")+"判断其他的值是否存在"+datalist.getString("cameraSn")+"其他的一些东西"+datalist.getString("hss"));
               if(log.getTableName().matches("db_dataCollection.t_cameraChannel")){
                   System.out.println("进入db_dataCollection.t_cameraChannel");
                  /**
                   * 将主键id赋值给它
                   */
                   List resultList = Lists.newArrayList();
                   DataMogodbDTO model = new DataMogodbDTO();
                  model.set_id(log.getPkId()!=null?log.getPkId():datalist.getString("_id"));
                  model.setPK(true);//更改标志位，设定为存在主键id
                  model.setCameraSn(datalist.getString("cameraSn")!=null?datalist.getString("cameraSn"):"");
                  model.setCameraStatus(datalist.getString("cameraStatus")!=null?datalist.getString("cameraStatus"):"");
                  model.setCameraName(datalist.getString("cameraName")!=null?datalist.getString("cameraName"):"");
                  model.setBlurStatus(datalist.getString("blurStatus")!=null?datalist.getString("blurStatus"):"");
                  model.setBrightnessStatus(datalist.getString("brightnessStatus")!=null?datalist.getString("brightnessStatus"):"");
                  model.setContrastStatus(datalist.getString("contrastStatus")!=null?datalist.getString("contrastStatus"):"");
                  model.setCoverStatus(datalist.getString("coverStatus")!=null?datalist.getString("coverStatus"):"");
                  model.setDitherStatus(datalist.getString("ditherStatus")!=null?datalist.getString("ditherStatus"):"");
                  model.setFrozenStatus(datalist.getString("frozenStatus")!=null?datalist.getString("frozenStatus"):"");
                  model.setLossStatus(datalist.getString("lossStatus")!=null?datalist.getString("lossStatus"):"");
                  model.setNoiseStatus(datalist.getString("noiseStatus")!=null?datalist.getString("noiseStatus"):"");
                  model.setScenechangeStatus(datalist.getString("scenechangeStatus")!=null?datalist.getString("scenechangeStatus"):"");
                  model.setStriationStatus(datalist.getString("striationStatus")!=null?datalist.getString("striationStatus"):"");
                  model.setUnbalanceStatus(datalist.getString("unbalanceStatus")!=null?datalist.getString("unbalanceStatus"):"");
                  model.setModifiedOn(datalist.getString("modifiedOn")!=null?datalist.getString("modifiedOn"):"");
                  resultList.add(model);
                  log.setDataJson(JSONObject.toJSONString(resultList));
              }else if(log.getTableName().matches("db_dataCollection.t_diskStatus")){
                   System.out.println("进入db_dataCollection.t_diskStatus");
                   List resultList = Lists.newArrayList();
                   DiskStatusMogodbDTO model2=new DiskStatusMogodbDTO();
                   model2.set_id(log.getPkId()!=null?log.getPkId():datalist.getString("_id"));
                   model2.setPK(true);//更改标志位，设定为存在主键id
                   model2.setDeviceMainType(datalist.getString("deviceMainType")!=null?datalist.getString("deviceMainType"):"");
                   model2.setDeviceName(datalist.getString("deviceName")!=null?datalist.getString("deviceName"):"");
                   model2.setDeviceStatus(datalist.getString("deviceStatus")!=null?datalist.getString("deviceStatus"):"");
                   model2.setDeviceType(datalist.getString("deviceType")!=null?datalist.getString("deviceType"):"");
                   model2.setModifiedOn(datalist.getString("modifiedOn")!=null?datalist.getString("modifiedOn"):"");
                   String  temp=datalist.getString("modifiedOn")!=null? datalist.getString("modifiedOn"):"--------------没有获取到任务数据";
                   System.out.println("输出对应传入到----------------------------------------------------------------------------------------------------------------------------------------------DTO中详细的变化的time的值"+temp);
                   resultList.add(model2);
                   log.setDataJson(JSONObject.toJSONString(resultList));
               }else{
                   logger.info("datasource data get fail");
               }
              log.setPkId(log.getPkId()!=null?log.getPkId():datalist.getString("_id"));//赋值主键id.
              //resultList.add(model);
              //log.setDataJson(JSONObject.toJSONString(resultList));
//            model.Firstkey();
//            model.setFiledName(column.getName());
//            model.setFiledType(column.getMysqlType());
//            model.setValue(column.getValue());
       // }else if(jsStr.getString("$v")!="1"){
             //进入数据的删除和新增操作

       // }


       // Iterator keys = keylist.keySet().iterator();
        //for (CanalEntry.Column column : columns) {
            //DataMogodbDTO model = new DataMogodbDTO();//数据同步传输表字段，进行数据的跟踪
//            model.Firstkey();
//            model.setFiledName(column.getName());
//            model.setFiledType(column.getMysqlType());
//            model.setValue(column.getValue());
            // 获取主键字段和值
//            if (column.getIsKey()) {
//                id = column.getValue();
//                model.setPK(true);  //这个是事件的主键Id，当存在的时候进行标志，不存在就算了
//            }

//        }

            // log.setPkId(id);
            // log.setDataJson(JSONObject.toJSONString(resultList));
        }
    }


/** 打印结果举例
 ================> binlog[mysql-bin.000010:2407] , name[wesafe_west,abproles] , eventType : UPDATE
 -------> before
 Id : 10    colType=int(11)    update=false
 ConcurrencyStamp : 1ae6614c-e82c-4746-a9a3-57cbc60e80bd    colType=longtext    update=false
 CreationTime : 2017-11-01 04:00:21.000000    colType=datetime(6)    update=false
 CreatorUserId : 2    colType=bigint(20)    update=false
 DeleterUserId : 2    colType=bigint(20)    update=false
 DeletionTime : 2018-02-23 17:54:36.063460    colType=datetime(6)    update=false
 DisplayName : 管理员    colType=varchar(64)    update=false
 IsDefault : 0    colType=bit(1)    update=false
 IsDeleted : 0    colType=bit(1)    update=false
 IsStatic : 0    colType=bit(1)    update=false
 LastModificationTime :     colType=datetime(6)    update=false
 LastModifierUserId :     colType=bigint(20)    update=false
 Name : 68f545a7addc42d29a16d60afa90d11a    colType=varchar(32)    update=false
 NormalizedName : 68F545A7ADDC42D29A16D60AFA90D11A    colType=varchar(32)    update=false
 TenantId : 1    colType=int(11)    update=false
 -------> after
 Id : 10    colType=int(11)    update=false
 ConcurrencyStamp : 1ae6614c-e82c-4746-a9a3-57cbc60e80bd    colType=longtext    update=false
 CreationTime : 2017-11-01 04:00:21.000000    colType=datetime(6)    update=false
 CreatorUserId : 2    colType=bigint(20)    update=false
 DeleterUserId : 2    colType=bigint(20)    update=false
 DeletionTime : 2018-02-23 17:54:36.063460    colType=datetime(6)    update=false
 DisplayName : 管理员1    colType=varchar(64)    update=true
 IsDefault : 0    colType=bit(1)    update=false
 IsDeleted : 0    colType=bit(1)    update=false
 IsStatic : 0    colType=bit(1)    update=false
 LastModificationTime :     colType=datetime(6)    update=false
 LastModifierUserId :     colType=bigint(20)    update=false
 Name : 68f545a7addc42d29a16d60afa90d11a    colType=varchar(32)    update=false
 NormalizedName : 68F545A7ADDC42D29A16D60AFA90D11A    colType=varchar(32)    update=false
 TenantId : 1    colType=int(11)    update=false

 ================> binlog[mysql-bin.000008:2838] , name[wesafe_west,user_profile] , eventType : INSERT
 -------> before
 -------> after
 uid : 211    update=true
 TenantId : 1    update=true
 ================> binlog[mysql-bin.000008:3122] , name[wesafe_west,user_profile] , eventType : DELETE
 -------> before
 uid : 211    update=false
 TenantId : 1    update=false
 -------> after

 */