kafka消息-----定时清理

项目中最近使用kafka需要定时清理消息，我们知道kafka有周期性清理消息机制,但是项目中往往因为数据量较大，需要手动控制分区已消费消息的清理。

此处使用的是反编译程序，具体几个方法已标出，

 个人清理想法：大致流程就是根据topic获取所有的分区，然后根据遍历每个分区的偏移量，然后存入map，存入时先判断偏移量大小，保留最小该分区消费者最小偏移量的值,然后删除该偏移量以前的数据，

下面的反码程序实际上没有做最小偏移量识别，直接用的分区最后一个最新的的偏移量，是通过创建一个消费者直接将其Offset  toEnd，然后删掉该分区该offset之前的消息，具体的大家可根据自己要求自行修改

 

package com.atguigu.kafkaclean;

import org.springframework.stereotype.*;
import org.springframework.context.annotation.*;
import com.ankki.kafkaclient.mapper.*;
import javax.annotation.*;
import java.time.format.*;
import com.ankki.kafkaclient.utils.*;
import java.util.concurrent.*;
import org.apache.kafka.clients.admin.*;
import org.springframework.scheduling.annotation.*;
import com.ankki.kafkaclient.consumer.*;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.*;
import com.ankki.kafkaclient.model.*;
import java.time.*;
import org.slf4j.*;
import java.util.*;
@Slf4j
@Component
@Configuration
@EnableScheduling
public class SaticScheduleTask
{
    private static final Logger log;
    private static final String SASL_VALUE = "yes";
    private static final String TOPIC;
    private static final String KAFKA_SERVER_ADDRR;
    @Resource
    private SysLogMapper sysLogMapper;
    
    @Scheduled(cron = "${cleanTime}")
    public void configureTasks() {
        log.info("\u6267\u884c\u5b9a\u65f6\u6e05\u7406\u4efb\u52a1\u65f6\u95f4: {}", (Object)LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
        final Properties props = new Properties();
        if ("yes".equals(PropertyUtils.getProperty("openSasl"))) {
            KafkaUtils.jaasConfig(props);
        }
        ((Hashtable)props).put("bootstrap.servers", SaticScheduleTask.KAFKA_SERVER_ADDRR);
        try (final AdminClient adminClient = KafkaAdminClient.create(props)) {
            final String[] topics = SaticScheduleTask.TOPIC.split(",");
            final Map recordsToDeleteMap = new HashMap(16);
            DeleteRecordsResult deleteRecordsResult = null;
            for (final String topic : topics) {
                final Map partitionInfoMap = this.getPartitionsForTopic(topic);
                for (final Map.Entry entry2 : partitionInfoMap.entrySet()) {
                    final TopicPartition topicPartition = new TopicPartition(topic, (int)entry2.getKey());
                    final RecordsToDelete recordsToDelete = RecordsToDelete.beforeOffset((long)entry2.getValue());
                    recordsToDeleteMap.put(topicPartition, recordsToDelete);
                }
            }
            SaticScheduleTask.log.debug("\u6e05\u7406\u96c6\u5408\uff1a{}", (Object)recordsToDeleteMap.toString());
            deleteRecordsResult = adminClient.deleteRecords((Map)recordsToDeleteMap);
            final Map> lowWatermarks = (Map>)deleteRecordsResult.lowWatermarks();
            final Exception ex;
            Exception e;
            lowWatermarks.entrySet().forEach(entry -> {
                try {
                    SaticScheduleTask.log.info("\u6e05\u7406\u6570\u636e\u4fe1\u606f \u4e3b\u9898\uff1a{} \u5206\u533a\uff1a{} \u6700\u65b0\u504f\u79fb\u91cf\uff1a{} \u7684\u7ed3\u679c\uff1a{}", new Object[] { entry.getKey().topic(), entry.getKey().partition(), ((DeletedRecords)((KafkaFuture)entry.getValue()).get()).lowWatermark(), ((KafkaFuture)entry.getValue()).isDone() });
                }
                catch (InterruptedException | ExecutionException ex2) {
                    e = ex;
                    SaticScheduleTask.log.error("\u83b7\u53d6kafka\u6e05\u7406\u6807\u8bb0\u7ed3\u679c\u5f02\u5e38\uff1a", (Throwable)e);
                }
                return;
            });
            this.recordSystemLog("true");
            SaticScheduleTask.log.debug("\u5b9a\u65f6\u6e05\u7406\u5df2\u6d88\u8d39\u7684\u6570\u636e\u6210\u529f");
        }
        catch (Exception e2) {
            this.recordSystemLog(null);
            SaticScheduleTask.log.error("kafka\u5b9a\u65f6\u6267\u884c\u6e05\u7406\u6807\u8bb0\u5f02\u5e38\uff1a", (Throwable)e2);
        }
    }
    
    private Consumer createConsumer() {
        final Consumer consumer = (Consumer)new KafkaConsumer(KafkaConsumerConfig.consumerConfigs());
        return consumer;
    }
    
    private Map getPartitionsForTopic(final String topic) {
        final Map partitionInfoMap = new HashMap(16);
        final Consumer consumer = this.createConsumer();
        final Collection partitionInfos = (Collection)consumer.partitionsFor(topic);
        final List tp = new ArrayList();
        final List list;
        final Consumer consumer2;
        final Map map;
        partitionInfos.forEach(str -> {
            list.add(new TopicPartition(topic, str.partition()));
            consumer2.assign((Collection)list);
            consumer2.seekToEnd((Collection)list);
            map.put(str.partition(), consumer2.position(new TopicPartition(topic, str.partition())));
            return;
        });
        return partitionInfoMap;
    }
    
    private void recordSystemLog(final String value) {
        final SysLog systemLog = new SysLog();
        systemLog.setLogTime(Instant.now().toEpochMilli());
        systemLog.setLogType((Byte)2);
        if ("true".equals(value)) {
            systemLog.setLogStatus((Byte)1);
            systemLog.setLogLevel((Byte)2);
            systemLog.setLogDesc("\u5b9a\u65f6\u6e05\u7406\u5df2\u6d88\u8d39\u7684\u6570\u636e\u6210\u529f\uff01");
        }
        else {
            systemLog.setLogStatus((Byte)0);
            systemLog.setLogLevel((Byte)1);
            systemLog.setLogDesc("\u5b9a\u65f6\u6e05\u7406\u5df2\u6d88\u8d39\u7684\u6570\u636e\u5931\u8d25\uff01");
        }
        this.sysLogMapper.insertSelective(systemLog);
    }
    
    static {
        log = LoggerFactory.getLogger((Class)SaticScheduleTask.class);
        TOPIC = PropertyUtils.getProperty("cleanTopics");
        KAFKA_SERVER_ADDRR = PropertyUtils.getProperty("bootstrapServers");
    }
}