flink学习笔记-flink的DataStream集成kafka

对于实时处理当中,我们实际工作当中的数据源一般都是使用kafka,所以我们一起来看看如何通过Flink来集成kafka。flink提供了一个特有的kafka connector去读写kafka topic的数据。flink消费kafka数据,并不是完全通过跟踪kafka消费组的offset来实现去保证exactly-once的语义,而是flink内部去跟踪offset和做checkpoint去实现exactly-once的语义,而且对于kafka的partition,Flink会启动对应的并行度去处理kafka当中的每个分区的数据

  • flink整合kafka官网介绍:https://ci.apache.org/projects/flink/flink-docs-release-1.6/dev/connectors/kafka.html

  • 第一步:导入jar包

    <dependency>
        <groupId>org.apache.flinkgroupId>
        <artifactId>flink-connector-kafka-0.11_2.11artifactId>
        <version>1.8.1version>
    dependency>
    <dependency>
        <groupId>org.apache.kafkagroupId>
        <artifactId>kafka-clientsartifactId>
        <version>1.1.0version>
    dependency>
    
    <dependency>
        <groupId>org.slf4jgroupId>
        <artifactId>slf4j-apiartifactId>
        <version>1.7.25version>
    dependency>
    
    <dependency>
        <groupId>org.slf4jgroupId>
        <artifactId>slf4j-log4j12artifactId>
        <version>1.7.25version>
    dependency>
    
  • 第二步:将kafka作为flink的source来使用

    import java.util.Properties
    import org.apache.flink.api.common.serialization.SimpleStringSchema
    import org.apache.flink.contrib.streaming.state.RocksDBStateBackend
    import org.apache.flink.streaming.api.CheckpointingMode
    import org.apache.flink.streaming.api.environment.CheckpointConfig
    import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
    import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011
    
    object FlinkKafkaSource {
    
      def main(args: Array[String]): Unit = {
        val env = StreamExecutionEnvironment.getExecutionEnvironment
        //隐式转换
        import org.apache.flink.api.scala._
        //checkpoint配置
        env.enableCheckpointing(100);
        env.getCheckpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
        env.getCheckpointConfig.setMinPauseBetweenCheckpoints(500);
        env.getCheckpointConfig.setCheckpointTimeout(60000);
        env.getCheckpointConfig.setMaxConcurrentCheckpoints(1);
        env.getCheckpointConfig.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
        env.setStateBackend(new RocksDBStateBackend("hdfs://node01:8020/flink/checkDir",true))
    
        val topic = "test"
        val prop = new Properties()
        prop.setProperty("bootstrap.servers","node01:9092")
        prop.setProperty("group.id","con1")
        prop.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        prop.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    
        var kafkaSoruce: FlinkKafkaConsumer011[String] = new FlinkKafkaConsumer011[String](topic, new SimpleStringSchema(), prop)
    
        kafkaSoruce.setCommitOffsetsOnCheckpoints(true)
        //设置statebackend
        env.setStateBackend(new RocksDBStateBackend("hdfs://node01:8020/flink_kafka/checkpoints",true));
        val result: DataStream[String] = env.addSource(kafkaSoruce)
        result.print()
        env.execute()
      }
    }
    
  • 第三步:将kafka作为flink的sink来使用

    import java.util.Properties
    import org.apache.flink.api.common.serialization.SimpleStringSchema
    import org.apache.flink.contrib.streaming.state.RocksDBStateBackend
    import org.apache.flink.streaming.api.CheckpointingMode
    import org.apache.flink.streaming.api.environment.CheckpointConfig
    import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
    import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011
    import org.apache.flink.streaming.connectors.kafka.internals.KeyedSerializationSchemaWrapper
    
    object FlinkKafkaSink {
      def main(args: Array[String]): Unit = {
          val env = StreamExecutionEnvironment.getExecutionEnvironment
          //隐式转换
          import org.apache.flink.api.scala._
          //checkpoint配置
          env.enableCheckpointing(5000);
          env.getCheckpointConfig.setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
          env.getCheckpointConfig.setMinPauseBetweenCheckpoints(500);
          env.getCheckpointConfig.setCheckpointTimeout(60000);
          env.getCheckpointConfig.setMaxConcurrentCheckpoints(1);
          env.getCheckpointConfig.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
          //设置statebackend
          env.setStateBackend(new RocksDBStateBackend("hdfs://node01:8020/flink_kafka_sink/checkpoints",true));
    
    
          val text = env.socketTextStream("node01",9000)
    
          val topic = "test"
          val prop = new Properties()
          prop.setProperty("bootstrap.servers","node01:9092")
          prop.setProperty("group.id","kafka_group1")
    
    
          //第一种解决方案,设置FlinkKafkaProducer011里面的事务超时时间
          //设置事务超时时间
          prop.setProperty("transaction.timeout.ms",60000*15+"");
          //第二种解决方案,设置kafka的最大事务超时时间
          //FlinkKafkaProducer011 myProducer = new FlinkKafkaProducer011<>(brokerList, topic, new SimpleStringSchema());
          //使用支持仅一次语义的形式
          val myProducer = new FlinkKafkaProducer011[String](topic,new KeyedSerializationSchemaWrapper[String](new SimpleStringSchema()), prop, FlinkKafkaProducer011.Semantic.EXACTLY_ONCE)
          text.addSink(myProducer)
          env.execute("StreamingFromCollectionScala")
      }
    }
    

你可能感兴趣的:(fiink)