flink - kafka producer 实现多分区发送数据到consumer，进行消费

1. 1. 先建立好分区的topic

kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic flinkwrite5

建立分区为3 ，并行度为3 的producer

2.然后在flink中连接这个topic

treamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "localhost:9092");

        DataStream stream = env.addSource(new SimpleStringGenerator());
        stream.addSink(new FlinkKafkaProducer010("flink_Producer", new SimpleStringSchema(), properties)).setParallelism(3);  
        env.execute() 

通过从写SourceFunction来实现数据的生产

 public static class StringInput implements SourceFunction {

        long i = 0;
        boolean swith = true;

        String s= "fngkjdgndfngdf";
        @Override
        public void run(SourceContext ctx) throws Exception {
            for(int k=0;k<5;k++) {
                ctx.collect("flink:"+s.substring(k,k+2) +" "+ s.substring(k,k+2));
               
            }
        }

        @Override
        public void cancel() {
            swith = false;
        }

    }

}

然后在新建一个类，用来消费这个producer

public class kafkaConsumer {

    public static void main(String args[]) throws Exception {


        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(3);
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers","localhost:9092");
        properties.setProperty("group.id","consumer-group");
        properties.setProperty("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
        properties.setProperty("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
        properties.setProperty("auto.offset.reset","latest");
        FlinkKafkaConsumer010 myConsumer = new FlinkKafkaConsumer010("flink_Producer", new SimpleStringSchema(), properties);
       // myConsumer.setStartFromEarliest();

        myConsumer.setStartFromLatest();
        Map specificStartOffsets = new HashMap<>();
        specificStartOffsets.put(new KafkaTopicPartition("flink_Producer", 0), 23L);
        specificStartOffsets.put(new KafkaTopicPartition("flink_Producer", 1), 31L);
        specificStartOffsets.put(new KafkaTopicPartition("flink_Producer", 2), 43L);

        myConsumer.setStartFromSpecificOffsets(specificStartOffsets);
        DataStream stream = env.addSource(myConsumer);

        stream.flatMap(new FlatMapFunction() {
            @Override
            public void flatMap(String value, Collector out)
                    throws Exception {
                for(String word: value.split(" ")){
                    out.collect(word);
                }
            }
        });

        Properties properties1 = new Properties();
        properties1.setProperty("bootstrap.servers","localhost:9092");

        stream.print();
        System.out.println("teset");


        env.execute("kafka sink");
    }

}