flink实现将kafka数据写入kafka（java）--入门级

1、安装kafka（mac版）：brew install kafka

2、启动zookeeper、kafka、flink：

zookeeper：/usr/local/Cellar/zookeeper/3.4.13/bin/zkServer start

kafka：/usr/local/Cellar/kafka/2.1.1/bin/kafka-server-start /usr/local/etc/kafka/server.properties

flink：/usr/local/Cellar/apache-flink/1.7.1/libexec/bin/start-cluster.sh (接着就可以进入web页面(http://localhost:8081/))

3、创建topic，名为tuzisir：

kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic tuzisir

4、新建flink（java版）项目：

mvn archetype:generate \
    -DarchetypeGroupId=org.apache.flink \
    -DarchetypeArtifactId=flink-quickstart-java \
    -DarchetypeVersion=1.7.0 \
    -DgroupId=net.qutoutiao.di \
    -DartifactId=flink-logserver-java \
    -Dversion=1.0 \
    -Dpackage=net.qutoutiao.di \
    -DinteractiveMode=false

并将以下依赖添加到pom.xml中

   org.apache.flink
   flink-core
   ${flink.version}


   org.apache.flink
   flink-connector-kafka_2.11
   1.7.2

代码如下：

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;

import java.util.Properties;

public class ProducerKafkaFlink {
    public static void main(String[] args) throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //从kafka读取数据
        Properties properties = new Properties();
        properties.setProperty("bootstrap.servers", "localhost:9092");
        properties.setProperty("group.id", "test-consumer-group");
        FlinkKafkaConsumer myConsumer = new FlinkKafkaConsumer<>("tuzisir", new SimpleStringSchema(), properties);
        DataStream stream = env.addSource(myConsumer);
        stream.print();

        //将结果写到kafka
        stream.addSink(new FlinkKafkaProducer<>(
                "localhost:9092",
                "student-write",
                new SimpleStringSchema()
        )).name("flink-connectors-kafka");

        env.execute("write to kafka");

    }
}