Flink压缩并按照指定字段分区发送Kafka

一. 工具类

/**
 * 功能描述: 

 * 〈按照指定的filed字段分区发送到kafka〉
 * @Param: [topic, filed]
 * @Return: org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer
 * @Author: sheng
 * @Date: 2022/4/28 10:16 上午
 */
    public static FlinkKafkaProducer<String> getGeneralKafkaPartitionProducer(String topic, String filed) {
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", KafkaConfig.bootstrapServers);
        properties.setProperty("group.id", "flink");
        properties.setProperty("compression.type", "lz4");
        properties.setProperty("compression.codec", "lz4");

        FlinkKafkaProducer<String> producer = new FlinkKafkaProducer<String>(
                topic,
                new SimpleStringSchema(),
                properties,
                Optional.of(new FlinkKafkaPartitioner<String>() {
                    @Override
                    public int partition(String record, byte[] key, byte[] value, String targetTopic, int[] partitions) {
                        JSONObject jsonObject = JSONObject.parseObject(record);
                        Object o = jsonObject.get(filed);
                        int par = Math.abs(o.hashCode() % partitions.length);
                        return par;
                    }
                }));

        return producer;
    }

二. 使用

// 前面业务逻辑省略
dockResultStream
                .map(new MapFunction<HiveChange, String>() {
                    @Override
                    public String map(HiveChange value) throws Exception {
                        return JSON.toJSONString(value);
                    }
                })
                .addSink(KafkaUtils.getGeneralKafkaPartitionProducer("hive.history.process", "id")); // 按照id字段进行分区