Spark2.3.0 通过Phoenix4.7 查询 Hbase 数据.
0. 环境
| 软件 | 版本 |
| JDK | 1.8 |
| SCALA | 2.11.8 |
| Spark | 2.3.0 |
| Phoenix | 4.7.0 |
| Hbase | 1.1.2 |
1. 目标数据
CREATE TABLE TABLE1 (ID BIGINT NOT NULL PRIMARY KEY, COL1 VARCHAR);
UPSERT INTO TABLE1 (ID, COL1) VALUES (1, 'test_row_1');
UPSERT INTO TABLE1 (ID, COL1) VALUES (2, 'test_row_2');
2. 代码:
package com.zl
import java.util.Properties
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.SparkContext
import org.apache.spark.sql.{Row, SQLContext, SparkSession}
/**
*
*
*/
object App {
private val DB_PHOENIX_DRIVER = "org.apache.phoenix.jdbc.PhoenixDriver"
private val DB_PHOENIX_URL = "jdbc:phoenix:hadoop002,hadoop004,hadoop003"
private val DB_PHOENIX_USER = ""
private val DB_PHOENIX_PASS = ""
private val DB_PHOENIX_FETCHSIZE = "10000"
/**
* 加载数据查询SQL
*/
private val SQL_QUERY = " ( SELECT ID , COL1 FROM TABLE1 limit 10 ) events "
def main(args: Array[String]): Unit = {
val conf = new SparkConf()
conf.setAppName("phoenix-test")
.setMaster("local[*]")
val sc = new SparkContext(conf)
val sparkSession = SparkSession.builder().config(conf).getOrCreate()
// JDBC连接属性
val connProp = new Properties
connProp.put("driver", DB_PHOENIX_DRIVER)
connProp.put("user", DB_PHOENIX_USER)
connProp.put("password", DB_PHOENIX_PASS)
connProp.put("fetchsize", DB_PHOENIX_FETCHSIZE)
val rows = sparkSession.read.jdbc(DB_PHOENIX_URL, SQL_QUERY, connProp).javaRDD
println(rows.collect())
}
}
3.项目结构与运行效果
4. 新建项目 pom.xml 文件:
4.0.0
org.apache.spark
spark-parent_2.11
2.3.2
../pom.xml
spark-core_2.11
core
jar
Spark Project Core
http://spark.apache.org/
org.apache.avro
avro
org.apache.avro
avro-mapred
${avro.mapred.classifier}
com.google.guava
guava
com.twitter
chill_${scala.binary.version}
com.twitter
chill-java
org.apache.xbean
xbean-asm5-shaded
org.apache.hadoop
hadoop-client
org.apache.spark
spark-launcher_${scala.binary.version}
${project.version}
org.apache.spark
spark-kvstore_${scala.binary.version}
${project.version}
org.apache.spark
spark-network-common_${scala.binary.version}
${project.version}
org.apache.spark
spark-network-shuffle_${scala.binary.version}
${project.version}
org.apache.spark
spark-unsafe_${scala.binary.version}
${project.version}
net.java.dev.jets3t
jets3t
org.apache.curator
curator-recipes
org.eclipse.jetty
jetty-plus
compile
org.eclipse.jetty
jetty-security
compile
org.eclipse.jetty
jetty-util
compile
org.eclipse.jetty
jetty-server
compile
org.eclipse.jetty
jetty-http
compile
org.eclipse.jetty
jetty-continuation
compile
org.eclipse.jetty
jetty-servlet
compile
org.eclipse.jetty
jetty-proxy
compile
org.eclipse.jetty
jetty-client
compile
org.eclipse.jetty
jetty-servlets
compile
javax.servlet
javax.servlet-api
${javaxservlet.version}
org.apache.commons
commons-lang3
org.apache.commons
commons-math3
com.google.code.findbugs
jsr305
org.slf4j
slf4j-api
org.slf4j
jul-to-slf4j
org.slf4j
jcl-over-slf4j
log4j
log4j
org.slf4j
slf4j-log4j12
com.ning
compress-lzf
org.xerial.snappy
snappy-java
org.lz4
lz4-java
com.github.luben
zstd-jni
org.roaringbitmap
RoaringBitmap
commons-net
commons-net
org.scala-lang
scala-library
org.json4s
json4s-jackson_${scala.binary.version}
org.glassfish.jersey.core
jersey-client
org.glassfish.jersey.core
jersey-common
org.glassfish.jersey.core
jersey-server
org.glassfish.jersey.containers
jersey-container-servlet
org.glassfish.jersey.containers
jersey-container-servlet-core
io.netty
netty-all
io.netty
netty
com.clearspring.analytics
stream
io.dropwizard.metrics
metrics-core
io.dropwizard.metrics
metrics-jvm
io.dropwizard.metrics
metrics-json
io.dropwizard.metrics
metrics-graphite
com.fasterxml.jackson.core
jackson-databind
com.fasterxml.jackson.module
jackson-module-scala_${scala.binary.version}
org.apache.derby
derby
test
org.apache.ivy
ivy
oro
oro
${oro.version}
org.seleniumhq.selenium
selenium-java
test
org.seleniumhq.selenium
selenium-htmlunit-driver
test
xml-apis
xml-apis
test
org.hamcrest
hamcrest-core
test
org.hamcrest
hamcrest-library
test
org.mockito
mockito-core
test
org.scalacheck
scalacheck_${scala.binary.version}
test
org.apache.curator
curator-test
test
net.razorvine
pyrolite
4.13
net.razorvine
serpent
net.sf.py4j
py4j
0.10.7
org.apache.spark
spark-tags_${scala.binary.version}
org.apache.spark
spark-launcher_${scala.binary.version}
${project.version}
tests
test
org.apache.spark
spark-tags_${scala.binary.version}
test-jar
test
org.apache.commons
commons-crypto
${hive.group}
hive-exec
provided
${hive.group}
hive-metastore
provided
org.apache.thrift
libthrift
provided
org.apache.thrift
libfb303
provided
target/scala-${scala.binary.version}/classes
target/scala-${scala.binary.version}/test-classes
${project.basedir}/src/main/resources
${project.build.directory}/extra-resources
true
org.apache.maven.plugins
maven-antrun-plugin
generate-resources
run
org.apache.maven.plugins
maven-dependency-plugin
copy-dependencies
package
copy-dependencies
${project.build.directory}
false
false
true
true
guava,jetty-io,jetty-servlet,jetty-servlets,jetty-continuation,jetty-http,jetty-plus,jetty-util,jetty-server,jetty-security,jetty-proxy,jetty-client
true
Windows
Windows
.bat
unix
unix
.sh
sparkr
org.codehaus.mojo
exec-maven-plugin
sparkr-pkg
compile
exec
${project.basedir}${file.separator}..${file.separator}R${file.separator}install-dev${script.extension}