kafka动态创建消费者

@Component
public class ConsumerContainer {

    protected final Logger logger = LoggerFactory.getLogger(this.getClass());

    /**
     * 使用map对象，便于根据topic存储对应的KafkaConsumer
     */
    private Map kafkaConsumerThreadMap = new HashMap<>();

    @Resource(name = "kafkaPropsConfig")
    private Map props;

    /**
     * 添加消费者
     * @param topic
     * @param consumer
     */
    public synchronized void addConsumer(String topic, Consumer consumer){
        if(kafkaConsumerThreadMap.get(topic)!=null){
            logger.warn("重复创建消费者：{}",topic);
        }
        KafkaConsumer stringStringKafkaConsumer = new KafkaConsumer<>(props);
        stringStringKafkaConsumer.subscribe(Arrays.asList(topic));
        ConsumerRecords poll = stringStringKafkaConsumer.poll(3000);
        //创建消费者线程
        KafkaConsumerThread kafkaConsumerThread = new KafkaConsumerThread(stringStringKafkaConsumer,consumer);
        kafkaConsumerThread.start();
        kafkaConsumerThreadMap.put(topic,kafkaConsumerThread);
        logger.info("创建消费者成功：{}",topic);
    }

    /**
     * 删除消费者
     * @param topic
     */
    public synchronized void deleteConsumer(String topic){
        KafkaConsumerThread kafkaConsumerThread = kafkaConsumerThreadMap.get(topic);
        if (kafkaConsumerThread == null) {
            logger.warn("该消费者已经被删除：{}",topic);
            return;
        }
        //打断消费者线程
        kafkaConsumerThread.interrupt();
        kafkaConsumerThreadMap.remove(topic);
        logger.info("消费者删除成功：{}",topic);
    }

    public synchronized List listConsumer(){
        List list = new ArrayList<>();
        for (String s : kafkaConsumerThreadMap.keySet()) {
            list.add(s);
        }
        return list;
    }

}

@Data
@AllArgsConstructor
@NoArgsConstructor
public class DataAcquistion {

    String id;

    String vision;

    String method;

    Map properties;
}

public interface DataAcquistionDao {

    List selectIEquipment();

    EquipmentEntity selectIEquipmentByCode(String code);

    EquipmentEntity selectIEquipmentByGid(String gid);

    DataSubmatrixEntity selectDataSubmatrixByDataObjId(String dataObjId);

    List selectLocalColumnByDsId(String DsId);
}

@Repository
public class DataAcquistionDaoImpl implements DataAcquistionDao {

    @Autowired
    private BaseDao baseDao;

    @Autowired
    private BaseDao localColumnBaseDao;

    @Autowired
    private BaseDao dataSubmatrixBaseDao;

    @Override
    public List selectIEquipment() {
        return baseDao.listByCriterion(EquipmentEntity.class);
    }

    @Override
    public EquipmentEntity selectIEquipmentByCode(String code) {
        return baseDao.get("rscode",code,EquipmentEntity.class);
    }

    @Override
    public EquipmentEntity selectIEquipmentByGid(String gid) {
        return baseDao.get(gid,EquipmentEntity.class);
    }

    @Override
    public DataSubmatrixEntity selectDataSubmatrixByDataObjId(String dataObjId) {
        return dataSubmatrixBaseDao.get("dataObjId",dataObjId,DataSubmatrixEntity.class);
    }

    @Override
    public List selectLocalColumnByDsId(String DsId) {
        return localColumnBaseDao.listByCriterion(LocalColumnEntity.class, Restrictions.eq("dataSubmatrixId",DsId));
    }
}

public interface DataAcquistionService {

    /**
     * 根据code订阅主题
     * @param code
     * @return
     */
    boolean createConsumerByCode(String code);

    /**
     * 订阅所有主题
     * @return
     */
    boolean createAllConsumer();

    /**
     * 查询所有未订阅的主题
     * @return
     */
    List listNoTopicConsumer();


    boolean createOtherConsumer();

    boolean getConsumerStaus(String code);

    boolean deleteAllTopic();

    boolean deleteTopicbyCode(String code);

    boolean deleteAllConsumer();

    boolean deleteConsumerByTopic(String code);

    boolean saveDataAcquistion(String topic, String message);


}

@Service
public class DataAcquistionServiceImpl implements DataAcquistionService {

    protected final Logger logger = LoggerFactory.getLogger(this.getClass());

    KafkaHelper kafkaHelper = ApplicationContextHelper.getBean(KafkaHelper.class);

    @Autowired
    private DataAcquistionDao dataAcquistionDao;

    @Autowired
    private IEquipmentService m_iEquipmentService;

    @Autowired
    private ITDMConfigService m_iTDMConfigService;

    @Autowired
    protected Map m_tProductDataSpaceDaoMap;

    @Autowired
    @Qualifier("ENTWareDeviceData")
    private IDataStorageService m_iDataStorageService;

    public static Map dictMap =new HashMap();
//    @Autowired
//    private IDataSubmatrixService m_iDataSubmatrixService;

    @PostConstruct
    protected void initKafkaConfig() {
        /*List tEquipment = m_iEquipmentService.getBusinessObj();
        for (IEquipment iEquipment : tEquipment) {
            if (iEquipment.getEntity().getRscode() == null){
                continue;
            }
            dictMap.put(iEquipment.getEntity().getRscode(),iEquipment.getEntity().getGID());
        }*/
        List equipmentEntities = dataAcquistionDao.selectIEquipment();
        for (EquipmentEntity equipmentEntity : equipmentEntities) {
            if (equipmentEntity.getRscode() == null){
                continue;
            }
            dictMap.put(equipmentEntity.getRscode(),equipmentEntity.getGID());
        }
        String strTemp = m_iTDMConfigService.getSysSignItem("RfSoft.TDMProduct.EquipmentKafka", "DefaultStart").getValue();
        if (strTemp.equals("true")){
            try {
                List list = kafkaHelper.list();
                for (String topic : list) {
                    kafkaHelper.addConsumer(topic,new PTVMateConsumer());
                }
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
    }

    @Override
    public boolean createConsumerByCode(String code) {
        if (StringUtils.isBlank(code)){
            logger.error("code为空，创建消费者失败");
            return false;
        }
        kafkaHelper.addConsumer("thing_"+code+"_property",new PTVMateConsumer());
        return true;
    }

    @Override
    public boolean createAllConsumer() {
        List tEquipment = m_iEquipmentService.getBusinessObj();
        for (IEquipment iEquipment : tEquipment) {
            String rscode = iEquipment.getEntity().getRscode();
            if (StringUtils.isBlank(rscode)){
                continue;
            }
            kafkaHelper.addConsumer("thing_"+rscode+"_property",new PTVMateConsumer());
        }
        return true;
    }

    @Override
    public List listNoTopicConsumer() {
        List list = kafkaHelper.listConsumer();
        List noConsumer = new ArrayList<>();
        List tEquipment = m_iEquipmentService.getBusinessObj();
        for (IEquipment iEquipment : tEquipment) {
            String rscode = iEquipment.getEntity().getRscode();
            if (StringUtils.isBlank(rscode)){
                continue;
            }
            if (!list.contains(rscode)){
                noConsumer.add("thing_"+rscode+"_property");
            }
        }
        return noConsumer;
    }

    @Override
    public boolean createOtherConsumer() {
        List list = listNoTopicConsumer();
        for (String s : list) {
            kafkaHelper.addConsumer(s,new PTVMateConsumer());
        }
        return true;
    }

    @Override
    public boolean getConsumerStaus(String code) {
        List list = listNoTopicConsumer();
        if (list.contains(code)){
            return false;
        }
        return true;
    }

    @Override
    public boolean deleteAllTopic() {
        try {
            List list = kafkaHelper.list();
            for (String s : list) {
                boolean b = kafkaHelper.deleteTopic(s);
                if (b == false){
                    return false;
                }
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return true;
    }

    @Override
    public boolean deleteTopicbyCode(String code) {
        return kafkaHelper.deleteTopic(code);
    }

    @Override
    public boolean deleteAllConsumer() {
        try {
            List list = kafkaHelper.list();
            for (String s : list) {
                kafkaHelper.deleteConsumer(s);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return true;
    }

    @Override
    public boolean deleteConsumerByTopic(String topic) {
        kafkaHelper.deleteConsumer(topic);
        return true;
    }

    @Override
    public boolean saveDataAcquistion(String topic,String message) {

        Object parse = JSONObject.parse(message);
        Map dataAcquistion = (Map) parse;
        Map map = (Map) dataAcquistion.get("properties");
//        List equipments = m_iEquipmentService.getBusinessObjByProperty("rscode",topic);
        String[] s = topic.split("_");
        String rscode = null;
        if (s.length>1){
            rscode = s[1];
        }else {
            return false;
        }
        String gid = dictMap.get(rscode);
        if (StringUtils.isBlank(gid)){
            EquipmentEntity equipmentEntity = dataAcquistionDao.selectIEquipmentByCode(topic);
//        if (equipments.size()!=1){
//            return false;
//        }
//        String gid = equipments.get(0).getEntity().getGID();
            gid = equipmentEntity.getGID();
        }
//        m_iDataStorageService.writeDataToDB();
       /* IDataSubmatrix iDataSubmatrix = m_iDataSubmatrixService.getBusinessObjByProperty("dataObjId", gid).stream().findFirst().orElse(null);
        List tLocalColumn = iDataSubmatrix.getChannel();
        List tField = tLocalColumn.stream().filter(t -> t.getEntity() != null && org.springframework.util.StringUtils.hasText(t.getEntity().getFieldName()) == true).
                map(t -> t.getEntity().getFieldName()).collect(Collectors.toList());*/
        List> tData = new ArrayList<>();
//        Map properties = dataAcquistion.getProperties();
        String strCurrTime = DateTime.getCurrDateTime("yyyy-MM-dd HH:mm:ss.SSS");
        Long lngCurrTime = DateTime.convertStringToLongTime(strCurrTime, "yyyy-MM-dd HH:mm:ss.SSS");
        map.put("DATETIMES", String.valueOf(lngCurrTime));

        /*
        for (String strField : tField) {
            if (StringUtils.isBlank(properties.get(strField))){
                properties.put(strField,null);
            }
        }*/
        tData.add(map);

        String strData = JSONString.object2Json(tData);
        boolean blnResult = false;
        try {
//            blnResult = m_iDataStorageService.writeDataToDB(strData, gid);
            blnResult = saveData(strData, gid,map);
            //super.printString(response, String.valueOf(blnResult));
        } catch (Exception ex) {
            ex.printStackTrace();
//            super.printString(response, ex.getMessage());
        }
        return blnResult;
        /*String strData = "";
        Map map = JSON.parseObject(dataAcquistion);
        Map map1 = new LinkedHashMap<>();
        for (Map.Entry stringObjectEntry : map.entrySet()) {
            if (!stringObjectEntry.getKey().equals("properties")) {
                map1.put(stringObjectEntry.getKey(), stringObjectEntry.getValue());
            } else {
                Map map2 = JSON.parseObject(stringObjectEntry.getValue().toString());
                for (Map.Entry objectEntry : map2.entrySet()) {
                    map1.put(objectEntry.getKey(), objectEntry.getValue());
                }
            }
        }
        strData = JSONString.object2Json(map1);
        boolean blnResult;
        try {
            blnResult = m_iDataStorageService.writeDataToDB(strData, map.get("id").toString());
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return blnResult;*/
    }
    protected static String m_strPKFieldName = "DATETIMES";
    protected static String m_strPKFieldType = "BIGINT";

    public boolean saveData(String strData, String strDataObjectId,Map map){

//        EquipmentEntity equipmentEntity = dataAcquistionDao.selectIEquipmentByGid(strDataObjectId);
        DataSubmatrixEntity dataSubmatrixEntity = dataAcquistionDao.selectDataSubmatrixByDataObjId(strDataObjectId);
        List localColumnEntities = dataAcquistionDao.selectLocalColumnByDsId(dataSubmatrixEntity.getGID());
//        Map parse = (Map) JSONObject.parse(strData);
        List keys = new ArrayList<>();
        for (String s : map.keySet()) {
            keys.add(s);
        }
        List tField = new ArrayList<>();
        for (LocalColumnEntity localColumnEntity : localColumnEntities) {
            if (keys.contains(localColumnEntity.getName())){
                tField.add(localColumnEntity.getName());
            }
        }
        List> tData = JSONString.json2MapList(strData);
        if (tData == null || tData.size() < 1)
        {
            logger.error("传入的JSON格式不正确，解析时出现问题。");
            return false;
        }

        Integer intDataRow = 0;
        List tDataKey = new ArrayList<>();
        //把传入的数据转换为指定的格式
        Map> iDataMap = new LinkedCaseInsensitiveMap<>();
        //只取数据库中已经存在的字段的值
        for (Map iItemMap: tData)
        {
            for (String strField : tField)
            {
                //检查传入的数据中有没有数据库的字段
                if (iItemMap.containsKey(strField))
                {
                    String strValue = iItemMap.get(strField).toString();
                    List