10-SparkSQL读取mysql数据源并将结果写回mysql
一、创建测试表t_user2、user_t和t_result
1、t_user2表结构如下：
CREATE TABLE `t_user2` (
`id` int(11) DEFAULT NULL COMMENT 'id',
`name` varchar(64) DEFAULT NULL COMMENT '用户名',
`password` varchar(64) DEFAULT NULL COMMENT '密码',
`age` int(11) DEFAULT NULL COMMENT '年龄'
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
2、user_t表结构如下：
CREATE TABLE `user_t` (
`id` int(11) DEFAULT NULL COMMENT 'id',
`name` varchar(64) DEFAULT NULL COMMENT '姓名',
`password` varchar(64) DEFAULT NULL COMMENT '密码',
`address` varchar(64) DEFAULT NULL COMMENT '地址',
`age` int(11) DEFAULT NULL COMMENT '年龄'
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
3、t_result表结构如下：
CREATE TABLE `t_result` (
`id` int(11) DEFAULT NULL COMMENT 'id',
`name` varchar(64) DEFAULT NULL COMMENT '姓名',
`password` varchar(64) DEFAULT NULL COMMENT '密码',
`address` varchar(64) DEFAULT NULL COMMENT '地址',
`age` int(11) DEFAULT NULL COMMENT '年龄'
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
4、插入测试数据：
INSERT INTO `t_user2` VALUES (12, 'cassie', '1234562', 25);
INSERT INTO `t_user2` VALUES (11, 'zhangs', '123456', 25);
INSERT INTO `t_user2` VALUES (23, 'zhangs', '2321312', 34);
INSERT INTO `t_user2` VALUES (22, 'tom', 'sadfdsa', 23);
INSERT INTO `user_t` VALUES (1, 'zhangs', '123456', NULL, 25);
INSERT INTO `user_t` VALUES (2, 'zhangs', '123456', NULL, 252);
二、创建maven工程，导入mysql驱动包、spark相关包
mysql-connector-java.5.1.24.jar
spark-assembly-1.6.2-hadoop2.6.0.jar
spark-examples-1.6.2-hadoop2.6.0.jar
注：pom.xml文件内容如下
4.0.0
com.itxiaobai
00-SparkSql
1.0-SNAPSHOT
io.netty
netty-all
4.1.18.Final
mysql
mysql-connector-java
5.1.47
com.google.code.gson
gson
2.8.5
junit
junit
4.12
org.apache.hadoop
hadoop-client
2.7.5
org.apache.hadoop
hadoop-common
2.7.5
org.apache.hadoop
hadoop-hdfs
2.7.5
org.apache.spark
spark-sql_2.11
2.3.0
org.apache.spark
spark-core_2.11
2.3.0
org.apache.spark
spark-core_2.11
2.3.0
org.scala-lang
scala-library
2.11.7
org.slf4j
slf4j-log4j12
1.7.10
commons-lang
commons-lang
2.5
commons-logging
commons-logging
1.1.3
三、创建本地执行的scala代码类：
SparkSqlMysqlDatasource.scala
package sql
import java.util.Properties
import org.apache.spark.sql.{DataFrame, SQLContext}
import org.apache.spark.{SparkConf, SparkContext}
/**
* 生产环境：下提交任务
* spark-submit --class sql.SparkSqlMysqlDatasource --master yarn-cluster --executor-memory 2G --num-executors 2 --driver-memory 1g --executor-cores 1 /data1/e_heyutao/sparktest/sparkEnn.jar
*
*/
object SparkSqlMysqlDatasource {
//数据库配置
lazy val url = "jdbc:mysql://your_ip:3306/my_test"
lazy val username = "root"
lazy val password = "secret_password"
def main(args: Array[String]) {
// val sparkConf = new SparkConf().setAppName("sparkSqlTest").setMaster("local[2]").set("spark.app.id", "sql")
val sparkConf = new SparkConf().setAppName("sparkSqlTest").setMaster("yarn-cluster").set("spark.app.id", "sqlTest")
//序列化
sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
sparkConf.set("spark.kryoserializer.buffer", "256m")
sparkConf.set("spark.kryoserializer.buffer.max", "2046m")
sparkConf.set("spark.akka.frameSize", "500")
sparkConf.set("spark.rpc.askTimeout", "30")
//获取context
val sc = new SparkContext(sparkConf)
//获取sqlContext
val sqlContext = new SQLContext(sc)
//引入隐式转换，可以使用spark sql内置函数
import sqlContext.implicits._
//创建jdbc连接信息
val uri = url + "?user=" + username + "&password=" + password + "&useUnicode=true&characterEncoding=UTF-8"
val prop = new Properties()
//注意：集群上运行时，一定要添加这句话，否则会报找不到mysql驱动的错误
prop.put("driver", "com.mysql.jdbc.Driver")
//加载mysql数据表
val df_test1: DataFrame = sqlContext.read.jdbc(uri, "user_t", prop)
val df_test2: DataFrame = sqlContext.read.jdbc(uri, "t_user2", prop)
//从dataframe中获取所需字段
df_test2.select("id", "name", "age").collect()
.foreach(row => {
println("id " + row(0) + " ,name " + row(1) + ", age " + row(2))
})
//注册成临时表
df_test1.registerTempTable("temp_table")
val total_sql = "select * from temp_table "
val total_df: DataFrame = sqlContext.sql(total_sql)
//将结果写入数据库中
val properties=new Properties()
properties.setProperty("user","root")
properties.setProperty("password","secret_password")
total_df.write.mode("append").jdbc("jdbc:mysql://your_ip:3306/my_test?useUnicode=true&characterEncoding=UTF-8","t_result",properties)
/**
* 注意：查看源码可以知道详细意思
def mode(saveMode: String): DataFrameWriter = {
this.mode = saveMode.toLowerCase match {
case "overwrite" => SaveMode.Overwrite
case "append" => SaveMode.Append
case "ignore" => SaveMode.Ignore
case "error" | "default" => SaveMode.ErrorIfExists
case _ => throw new IllegalArgumentException(s"Unknown save mode: $saveMode. " +
"Accepted modes are 'overwrite', 'append', 'ignore', 'error'.")
}
*/
//分组后求平均值
total_df.groupBy("name").avg("age").collect().foreach(x => {
println("name " + x(0))
println("age " + x(1))
})
}
}
结果：
id 12 ,name cassie, age 25
id 11 ,name zhangs, age 25
id 23 ,name zhangs, age 34
id 22 ,name tom, age 23
name zhangs
age 138.5
四、查看数据库表t_result，发现刚才从mysql中读取出来的数据已经插入到表中
spark相关文档
spark学习总结