Flink_DataStream 的 Transformation

DataStream 的 Transformation

1. KeyBy

逻辑上将一个流分成不相交的分区，每个分区包含相同键的元素。在内部，这是通过散 列分区来实现的

package com.czxy.flink.stream.transformation

import org.apache.flink.api.java.tuple.Tuple
import org.apache.flink.streaming.api.scala.{DataStream, KeyedStream, StreamExecutionEnvironment}

//keyBy分组操作算子
object StreamKeyBy {
  def main(args: Array[String]): Unit = {
    //1.创建执行环境
     val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    //2.构建数据集
    import org.apache.flink.api.scala._
    val elementSource: DataStream[String] = env.fromElements("hadoop hadoop spark hive flink flink")
    //3.数据组合成元祖类型
    val wordAndOne: DataStream[(String, Int)] = elementSource.flatMap(x=>x.split(" ")).map((_,1))
    //4.进行分组
    val KeyedStream: KeyedStream[(String, Int), Tuple] = wordAndOne.keyBy(0)
    //5.聚合计算
    val result: DataStream[(String, Int)] = KeyedStream.reduce((v1,v2)=>(v1._1,v1._2+v2._2))
    //6.打印输出
    result.print().setParallelism(1)
    //7.执行程序
    env.execute("StreamKeyBy")
  }
}

2. Connect

用来将两个 dataStream 组装成一个 ConnectedStreams 而且这个 connectedStream 的组成结构就是保留原有的 dataStream 的结构体；这样我们 就可以把不同的数据组装成同一个结构

代码示例：

package com.czxy.flink.stream.transformation

import org.apache.flink.streaming.api.functions.source.SourceFunction
import org.apache.flink.streaming.api.scala.{ConnectedStreams, DataStream, StreamExecutionEnvironment}
import org.apache.flink.api.scala._
/*** Connect合并流 */
object StreamConnect {
  
  def main(args: Array[String]): Unit = {
    
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    
    val source1: DataStream[Long] = env.addSource(new NoParallelSource()).setParallelism(1)
    val source2: DataStream[Long] = env.addSource(new NoParallelSource()).setParallelism(1)
    
    val connectedStreams: ConnectedStreams[Long, Long] = source1.connect(source2)
    val result: DataStream[String] = connectedStreams.map(item1 => {
      "item1: " + item1
    },
      item2 => {
        "item2: " + item2
      })
    result.print()
    env.execute("StreamConnect")
  }


  //实现一个单线程的，数据从1开始递增的数据集
  class NoParallelSource extends SourceFunction[Long]() {

    var number: Long = 1L
    var isRunning: Boolean = true

    override def run(ctx: SourceFunction.SourceContext[Long]): Unit = {
      while (isRunning) {
        ctx.collect(number)
        number += 1
        Thread.sleep(1)
        if (number > 5) {
          cancel()
        }
      }
    }

    override def cancel(): Unit = {
      isRunning = false
    }
  }
}

3. Split 和 select

代码实现

package com.czxy.flink.stream.transformation

import org.apache.flink.streaming.api.scala.{DataStream, SplitStream, StreamExecutionEnvironment}
import org.apache.flink.api.scala._
/**
 * 需求：
 * 给出数据 1, 2, 3, 4, 5, 6, 7
 * 请使用 split 和 select 把数据中的奇偶数分开， 并打印出奇数
 */
object StreamSplit {
  
  def main(args: Array[String]): Unit = {
    
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    
    val source: DataStream[Int] = env.fromElements(1, 2, 3, 4, 5, 6, 7)
    
    val splitStream: SplitStream[Int] = source.split(x => {
      (x % 2) match {
        case 0 => List("偶数")
        case 1 => List("奇数")
      }
    })
    val result: DataStream[Int] = splitStream.select("奇数")
    result.print()
    env.execute("StreamSplit")
  }
}

数据输出 Data Sinks

• 3 、将数据 sink 到本地文件(参考批处理)
• 4 、Sink 到本地集合(参考批处理)
• 5 、Sink 到 HDFS(参考批处理)

1.sink 到 kafka

代码示例

package com.czxy.flink.stream.sink

import java.util.Properties

import org.apache.flink.streaming.api.datastream.DataStreamSink
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011
import org.apache.flink.streaming.util.serialization.SimpleStringSchema

object StreamKafkaSink {

  def main(args: Array[String]): Unit = {
    //1.创建执行环境
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
    //2.构建数据集
    import org.apache.flink.api.scala._
    val source: DataStream[String] = env.fromElements("1,小丽,北京,女")
    //3.设置kafka的配置信息
    val topic="test"
    val properties: Properties = new Properties()
    properties.setProperty("bootstrap.servers","node01:9092")

    val flinkKafkaProducer: FlinkKafkaProducer011[String] = new FlinkKafkaProducer011[String](topic,new SimpleStringSchema(),properties)

    val result: DataStreamSink[String] = source.addSink(flinkKafkaProducer)

//    source.addSink(flinkKafkaProducer)
//    source.print()
    env.execute("StreamKafkaSink")
  }
}

2. sink 到 mysql

package com.czxy.flink.stream.sink

import java.sql.{Connection, DriverManager, PreparedStatement}

import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.datastream.DataStreamSink
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction
import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
import org.apache.flink.api.scala._

object StreamMysqlSink {

  case class Student(stuId: Int, stuName: String, stuAddr: String, stuSex: String)

  def main(args: Array[String]): Unit = {
    //1.创建执行环境
    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment

    //2.准备数据
    val source: DataStream[Student] = env.fromElements(
      //Student(8, "小青", "广州", "女"),
      Student(9, "wangman", "beijing", "nv")
    )
    val result: DataStreamSink[Student] = source.addSink(new MysqlSink())
    env.execute("StreamMysqlSink")

  }

  class MysqlSink extends RichSinkFunction[Student]() {
    var connection: Connection = null
    var ps: PreparedStatement = null

    override def open(parameters: Configuration): Unit = {
      val driver = "com.mysql.jdbc.Driver"
      val url = "jdbc:mysql://localhost:3306/test?characterEncoding=utf-8&useSSL=false"
      val username = "root"
      val password = "root"
      //1:加载驱动
      Class.forName(driver)
      //2：创建连接
      connection = DriverManager.getConnection(url, username, password)
      val sql =
        """
          |insert into student(id , name , addr , sex)values(?,?,?,?);
          |""".stripMargin
      //3:获得执行语句
      ps = connection.prepareStatement(sql)
    }
//    //关闭连接操作
//    override def close(): Unit = {
//      if (connection != null) {
//        connection.close() 
//      }
//      if (ps != null) {
//        ps.close()
//      }
//    }

    //每个元素的插入，都要触发一次 invoke，这里主要进行 invoke 插入 每条数据执行一次
    override def invoke(value: Student): Unit = {
      try{
        //4.组装数据，执行插入操作
        ps.setInt(1, value.stuId)
        ps.setString(2, value.stuName)
        ps.setString(3, value.stuAddr)
        ps.setString(4, value.stuSex)
        ps.executeUpdate()
      }catch{
        case e:Exception=>println(e.getMessage)
      }
    }
  }
}