把SparkStreaming版本的wordcount写入mysql数据库

1. package sparkstreamday01.SparkStream
3. import java.sql.{Connection, DriverManager, PreparedStatement, ResultSet}
5. import org.apache.log4j.{Level, Logger}
6. import org.apache.spark.SparkConf
7. import org.apache.spark.rdd.RDD
8. import org.apache.spark.streaming.dstream.ReceiverInputDStream
9. import org.apache.spark.streaming.{Seconds, StreamingContext}
11. object StreamRddsql3 {
12. //设置日志级别WARN
13. Logger.getLogger("org").setLevel(Level.WARN)
15. def main(args: Array[String]): Unit = {
16. //本地运行设置
17. val conf: SparkConf = new SparkConf()
18. .setMaster("local[3]") //至少两个核（运行一个,接收一个）
19. .setAppName(this.getClass.getSimpleName)
20. //StreamingContext
21. val ssc = new StreamingContext(conf, Seconds(4))
22. //接受socket信息 创建Dstream
23. val text: ReceiverInputDStream[String] = ssc.socketTextStream("Linux00", 9999)
24. //转换成RDD
25. text.foreachRDD(rdd => {
26. val result: RDD[(String, Int)] = rdd.flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _)
27. result.foreach(println)
28. //写入数据库
29. result.foreachPartition(p => {
30. val conn: Connection = DriverManager.getConnection("jdbc:mysql://localhost:3306/wjy1?characterEncoding=utf-8", "wjy1", "351991")
31. val statement: PreparedStatement = conn.prepareStatement("create table if not exists wordcount3 (word varchar(20),total int)")
32. statement.executeUpdate()
33. //创建连接
34. p.foreach(t => {
35. //如果插入的单词已经存在数据库中，则更新这个单词的值（当前出现的次数+数据库当前的次数）
36. val statement1: PreparedStatement = conn.prepareStatement("select * from wordcount3 where word=?")
37. statement1.setString(1, t._1)
38. val set: ResultSet = statement1.executeQuery()
39. if (set.next()) {
40. val count: Int = set.getInt("total")
41. val newcount: Int = count + t._2
42. //更新
43. val statement2: PreparedStatement = conn.prepareStatement("update wordcount3 set total=? where word=?")
44. statement2.setInt(1,newcount)
45. statement2.setString(2,t._1)
46. statement2.executeUpdate()
47. }else{
48. val statement3: PreparedStatement = conn.prepareStatement("insert into wordcount3 values(?,?)")
49. statement3.setString(1,t._1)
50. statement3.setInt(2,t._2)
51. statement3.executeUpdate()
52. }
53. })
54. conn.close()
55. })
57. })
58. //启动主程序
59. ssc.start()
60. //阻塞 等待程序被关闭
61. ssc.awaitTermination()
62. }
63. }