spark 写入dataframe到hbase

spark 写入dataframe到hbase

import org.apache.spark.sql.SparkSession
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase._
import org.apache.hadoop.hbase.client._
import org.apache.hadoop.hbase.client.Result
import org.apache.hadoop.hbase.client.Put
import org.apache.hadoop.hbase.mapreduce.TableOutputFormat
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.spark.sql._
import org.apache.spark.sql.Column
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._

//读取Hdfs数据写入hbase
object Hdfs_To_Hbase{
  def main(args:Array[String]):Unit={
	val spark = SparkSession.builder()
		.appName("Spark Write data to Hbase")
		.config("spark.some.config.option", "some-value")
		.config("spark.debug.maxToStringFields", "100")
		.config("spark.eventLog.enabled", "true")
		//.config("spark.sql.shuffle.partitions", "300") 
		//.config("spark.default.parallelism", "4") 
		.getOrCreate()
	import spark.implicits._
	//读取数据，参数选择格式推断,自动判断字段的数据类型
    val o_Sense = spark.read.format("csv").
	  option("sep", ",").
	  option("inferSchema", "true").
	  option("header", "true").
	  load("hdfs://nameservice1/test/test.csv")
	  //repartition(4)
    //cols = df1.columns.drop(4).map(f => col(f).cast("Float"))    //对前四列之外的列进行格式转换
	//val o_Sense1 = o_Sense.na.drop(40, cols=o_Sense.columns.drop(4))  //指定cols的非空值个数小于40，则删除此行数据
	val o_Sense2 = o_Sense.na.fill(0, cols=o_Sense.columns.drop(3))  //对cols列用0填补空值
	//o_Sense2.createOrReplaceTempView("Sense")
	val o_Sense3 = o_Sense2.withColumn("telephone", $"telephone"+1010101010)   //加密电话号码，字段加上1010101010
	o_Sense3.show()
	//写入hbase
	//链接hbase，相关参数配置
	val hbaseConf =HBaseConfiguration.create()
	hbaseConf.set(TableOutputFormat.OUTPUT_TABLE, "o_Sense")
	hbaseConf.set("mapreduce.output.fileoutputformat.outputdir", "/tmp")
	val job = Job.getInstance(hbaseConf)
	job.setOutputKeyClass(classOf[ImmutableBytesWritable])
	job.setOutputValueClass(classOf[Put])
	job.setOutputFormatClass(classOf[TableOutputFormat[ImmutableBytesWritable]])
	val rdd1 = o_Sense3.rdd.map(p=>{
	  val put=new Put(Bytes.toBytes(p(1).toString+"_"+p(0).toString)) //联合主键
	  put.addColumn(Bytes.toBytes("Sense"),Bytes.toBytes("live_cells"),Bytes.toBytes(p(2).toString))   
	  put.addColumn(Bytes.toBytes("Sense"),Bytes.toBytes("value1"),Bytes.toBytes(p(3).toString))
	  put.addColumn(Bytes.toBytes("Sense"),Bytes.toBytes("value2"),Bytes.toBytes(p(4).toString))
	  (new ImmutableBytesWritable(), put)}
	)
	rdd1.saveAsNewAPIHadoopDataset(job.getConfiguration)
  }
}