概述
之前写过spark批量导入Hbase的案例：Spark、BulkLoad Hbase、单列、多列，实现了多列的操作。整个过程涉及到排序、分解等操作相对复杂。
最近看官网的文档,发现有两种方法：
73节的Bulk Loading中的为我之前实现的方法
111节的Bulk Load为hbase-spark中自带的方法
但是在测试过程中发现官网的案例缺少某些关键代码，无法直接测试，于是花了一点时间去实现一下对比两种方法的效率
依赖包
UTF-8
2.11
2.3.2.3.1.0.0-78
2.0.2
3.1.1
org.scala-lang
scala-library
2.11.7
org.apache.spark
spark-sql_2.11
${spark.version}
org.apache.spark
spark-core_2.11
${spark.version}
org.apache.spark
spark-streaming_2.11
${spark.version}
org.apache.hadoop
hadoop-client
${hadoop.version}
com.alibaba
fastjson
1.2.45
org.apache.hbase
hbase-it
${hbase.version}
org.apache.hbase
hbase-spark
2.0.2.3.1.0.0-78
org.apache.hbase
hbase-spark-it
2.0.2.3.1.0.0-78
mysql
mysql-connector-java
5.1.38
实现代码
import java.io.IOException
import java.util.UUID
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.hadoop.hbase.client._
import org.apache.hadoop.hbase.io.ImmutableBytesWritable
import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2
import org.apache.hadoop.hbase.spark.{ByteArrayWrapper, FamiliesQualifiersValues, FamilyHFileWriteOptions, HBaseContext}
import org.apache.hadoop.hbase.tool.LoadIncrementalHFiles
import org.apache.hadoop.hbase.util.Bytes
import org.apache.hadoop.hbase.{HBaseConfiguration, HConstants, KeyValue, TableName}
import org.apache.hadoop.mapreduce.Job
import org.apache.spark.SparkContext
import org.apache.spark.sql.{DataFrame, SparkSession}
object BulkLoadDemo2 {
var sourceTable: String = "HFlileOut"
val spark: SparkSession = SparkSession
.builder()
.master("local")
.appName("ExportToHBase")
.config("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
.config("spark.shuffle.file.buffer", "2048k")
.config("spark.executor.cores", "2")
.getOrCreate()
val sc: SparkContext = spark.sparkContext
val conf: Configuration = HBaseConfiguration.create()
conf.set("hbase.zookeeper.quorum", "node1:2181,node2:2181,node3:2181")
conf.setInt("zookeeper.recovery.retry", 0)
conf.setInt("hbase.client.retries.number", 0)
val hbaseContext = new HBaseContext(sc, conf)
var savePath: Path = new Path(s"/warehouse/data/tmp/hbase/HFlileOut")
val cf: String = "cf"
val conn: Connection = ConnectionFactory.createConnection(conf)
val tableName: TableName = createTable(conn, sourceTable, cf)
def main(args: Array[String]): Unit = {
val start = System.currentTimeMillis()
//生成HFile
generateHFiles
//将HFile导入Hbase
loadHFileToHbase
val end = System.currentTimeMillis()
println("耗时： " + (end - start) / (1000 * 60).toDouble + " min...")
}
/**
* 生成hbase table的rowkey
* 随机加个UUID
*
* @param baseInfo rowkey中的组成部分
* @return
*/
def generateRowKey(baseInfo: String): String = {
val uuid = UUID.randomUUID.toString
val idx = uuid.lastIndexOf("-")
new StringBuffer(baseInfo).append("_").append(uuid.substring(idx + 1)).toString
}
/**
* 读取文件并生成hfile
*/
def generateHFiles(): Unit = {
//先删除可能存在的目录
delete_hdfspath(savePath.toUri.getPath)
//一定要导入这个包才能使用hbaseBulkLoadThinRows
import org.apache.hadoop.hbase.spark.HBaseRDDFunctions.GenericHBaseRDDFunctions
//获取数据
val sourceDataFrame: DataFrame = ...
val columnsName: Array[String] = sourceDataFrame.columns //获取所有列名
sourceDataFrame
.rdd
.map(row => {
val familyQualifiersValues: FamiliesQualifiersValues = new FamiliesQualifiersValues
val rowkey: String = generateRowKey(row.getAs[Int]("UID") + "")
//对每一列进行处理
for (i <- 0 until columnsName.length - 1) {
try {
familyQualifiersValues += (Bytes.toBytes(cf), Bytes.toBytes(columnsName(i)), Bytes.toBytes(row.getAs[String](columnsName(i)) + ""))
} catch {
case e: ClassCastException =>
familyQualifiersValues += (Bytes.toBytes(cf), Bytes.toBytes(columnsName(i)), Bytes.toBytes(row.getAs[BigInt](columnsName(i)) + ""))
case e: Exception =>
e.printStackTrace()
}
}
(new ByteArrayWrapper(Bytes.toBytes(rowkey)), familyQualifiersValues)
}).hbaseBulkLoadThinRows(hbaseContext, tableName, t => t, savePath.toUri.getPath, new java.util.HashMap[Array[Byte], FamilyHFileWriteOptions], compactionExclude = false, HConstants.DEFAULT_MAX_FILE_SIZE)
}
/**
* 创建HBase表
*
* @param conn
* @param tableName
* @param cf
* @return
*/
def createTable(conn: Connection, tableName: String, cf: String): TableName = {
val tn = TableName.valueOf(tableName)
var admin: Admin = null
try {
admin = conn.getAdmin
val flag = admin.tableExists(tn)
if (!flag) { //表不存在，则创建
val builder = TableDescriptorBuilder.newBuilder(tn)
val cfd: ColumnFamilyDescriptor = ColumnFamilyDescriptorBuilder.of(cf)
builder.setColumnFamily(cfd)
admin.createTable(builder.build)
}
} catch {
case e: IOException =>
e.printStackTrace()
}
tn
}
/**
* 将File文件导入HBase，本质是移动HFile到HBase目录下
*/
def loadHFileToHbase() = {
//开始即那个HFile导入到Hbase,此处都是hbase的api操作
val load: LoadIncrementalHFiles = new LoadIncrementalHFiles(conf)
//创建hbase的链接,利用默认的配置文件,实际上读取的hbase的master地址
val conn: Connection = ConnectionFactory.createConnection(conf)
//根据表名获取表
val table: Table = conn.getTable(TableName.valueOf(sourceTable))
//获取hbase表的region分布
val regionLocator: RegionLocator = conn.getRegionLocator(TableName.valueOf(sourceTable))
//创建一个hadoop的mapreduce的job
val job: Job = Job.getInstance(conf)
//设置job名称
job.setJobName(s"$sourceTable LoadIncrementalHFiles")
//此处最重要,需要设置文件输出的key,因为我们要生成HFil,所以outkey要用ImmutableBytesWritable
job.setMapOutputKeyClass(classOf[ImmutableBytesWritable])
//输出文件的内容KeyValue
job.setMapOutputValueClass(classOf[KeyValue])
//配置HFileOutputFormat2的信息
HFileOutputFormat2.configureIncrementalLoad(job, table, regionLocator)
//开始导入
load.doBulkLoad(savePath, conn.getAdmin, table, regionLocator)
spark.stop()
}
/**
* 删除hdfs下的文件
*
* @param url 需要删除的路径
*/
def delete_hdfspath(url: String) {
val hdfs: FileSystem = FileSystem.get(new Configuration)
val path: Path = new Path(url)
if (hdfs.exists(path)) {
hdfs.delete(path, true)
}
}
}
效率对比
对比两种方法，在导入hbase方面其实是一样的，不同点在于生成HFile的过程。
此方法不需要手动进行rowkey的排序，其内部已经做了该步骤。整体开发难度大幅降低。
开发效率提高了，但是对比相同资源、数据量的情况，执行效率如下：
方法一：生成HFile+导入Hbase耗时：53min
方法二：生成HFile耗时：63min
目前没经过太多的测试，未发现效率低的原因，后续跟踪看看
遇到问题
Could not access type Logging in package org.apache.spark
Error:scalac: missing or invalid dependency detected while loading class file 'HBaseContext.class'.
Could not access type Logging in package org.apache.spark,
because it (or its dependencies) are missing. Check your build definition for
missing or conflicting dependencies. (Re-run with `-Ylog-classpath` to see the problematic classpath.)
A full rebuild may help if 'HBaseContext.class' was compiled against an incompatible version of org.apache.spark.
经过查看HBase官网的一个JIRA找到问题所在
JIRA中的描述为：
在代码中引用了HBaseContext时，使用Spark2编译Spark应用程序将会失败，因为HBaseContext模块引用了org.apache.spark.Logging。
在Spark2中，由于Logging被移动到一个私有的包(org.apache.spark.internal)下导致。
解决办法：
-
在自己的工程下创建一个org.apache.spark的包 并在该报包下创建Trait类型的Logging.scala类型
image.png -
将spark-core工程下org.apache.spark.internal.Logging类内容拷贝至我们工程下创建的org.apache.spark.Logging类中
image.png
image.png
-
Could not access term streaming in package org.apache.spark
经过上述步骤后再次执行出现新的异常：
Error:scalac: missing or invalid dependency detected while loading class file 'HBaseContext.class'.
Could not access term streaming in package org.apache.spark,
because it (or its dependencies) are missing. Check your build definition for
missing or conflicting dependencies. (Re-run with `-Ylog-classpath` to see the problematic classpath.)
A full rebuild may help if 'HBaseContext.class' was compiled against an incompatible version of org.apache.spark.
问题类似，但是不再是Logging找不到，经过度娘找到一个方案Issue when trying to build #38
根据提示新增spark-streaming_2.11的依赖
org.apache.spark
spark-streaming_2.11
${spark.version}
暂时没搞明白为啥要加spark-streaming_2.11但是问题确实解决了