Flink读取数据存入Hbase

Flink读取数据存入Hbase

1.导入pom文件

 <dependency>
            <groupId>org.apache.flinkgroupId>
            <artifactId>flink-scala_2.11artifactId>
            <version>1.7.2version>


        dependency>
        
        <dependency>
            <groupId>org.apache.flinkgroupId>
            <artifactId>flink-streaming-scala_2.11artifactId>
            <version>1.7.2version>


        dependency>

        <dependency>
            <groupId>org.apache.hbasegroupId>
            
            <artifactId>hbase-shaded-clientartifactId>
            <version>1.3.1version>
        dependency>
        <dependency>
            <groupId>org.apache.hbasegroupId>
            <artifactId>hbase-serverartifactId>
            <version>1.3.1version>
        dependency>

2.编写scala代码

package org.zsw.flink

import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.flink.streaming.api.scala._
import org.apache.hadoop.conf
import org.apache.hadoop.hbase.{HBaseConfiguration, HConstants, TableName}
import org.apache.hadoop.hbase.client._

/**
 * @description:
 * @author: zsw
 * @date: Created in 2020/4/13 15:01
 * @modified By:
 */
object Flink_HbaseSink {

  def main(args: Array[String]): Unit = {

    val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment

    env.setParallelism(1)

    val dataStream: DataStream[String] = env.readTextFile("input/sensor.txt")
    val sendorDataStream: DataStream[SensorReading] = dataStream.map(data => {
      val dataArr: Array[String] = data.split(",")
      SensorReading(dataArr(0).trim, dataArr(1).trim.toLong, dataArr(2).trim.toDouble)
    })
    sendorDataStream.addSink(new MyHbaseSink())

    env.execute()

  }

}


class MyHbaseSink() extends  RichSinkFunction[SensorReading]{

//  创建连接
  var conn: Connection = _
//  创建BufferedMutator
//  作用类似put，可以实现批量异步
  var mutator:BufferedMutator = null

  override def open(parameters: Configuration): Unit = {
    val config: conf.Configuration = HBaseConfiguration.create()

//    设置zookeeper主机名
    config.set(HConstants.ZOOKEEPER_QUORUM,"spark01,spark02,spark03")
//    设置端口
    config.set(HConstants.ZOOKEEPER_CLIENT_PORT,"2181")
//    实例化
      conn = ConnectionFactory.createConnection(config)

    val tableName: TableName = TableName.valueOf("test")

    val params = new BufferedMutatorParams(tableName)

     mutator = conn.getBufferedMutator(params)

  }

  override def invoke(value: SensorReading, context: SinkFunction.Context[_]): Unit = {
    val family = "info"
    val put = new Put(value.id.getBytes())
    put.addColumn(family.getBytes(),"eventTime".getBytes(),value.timeStamp.toString.getBytes())
    put.addColumn(family.getBytes(),"temp".getBytes(),value.temperature.toString.getBytes())
    mutator.mutate(put)
  }

  override def close(): Unit = {
    mutator.close()
    conn.close()
  }
}