sparkStreamMySql

import java.sql.DriverManager

import com.typesafe.config.ConfigFactory
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}

object SparkStreamMySql {
  def main(args: Array[String]): Unit = {
    val config = ConfigFactory.load()
    val conf = new SparkConf().setMaster("local[2]").setAppName(this.getClass.getSimpleName)
    val ssc = new StreamingContext(conf,Seconds(3));
    val InputStream = ssc.socketTextStream("b02master",1315);
        InputStream.foreachRDD(rdd=>{
          val resultRdd = rdd.flatMap(_.split(" +")).map((_,1)).reduceByKey(_+_)
            //判断RDD是否为空
          if(resultRdd!=null){
            resultRdd.foreachPartition(iter=>{
              //每一个分区建立一次数据库连接
              //建立数据库连接
              val conn = DriverManager.getConnection(config.getString("db.url"),config.getString("db.user"),config.getString("db.password"))
              val table = config.getString("db.table")
              val createTable = conn.prepareStatement(s"create table if not exists  ${table} (word varchar(15),cnts int)")
              createTable.execute()
              iter.foreach(tp=>{
                //先查询表里是否有数据
                val selectStatus = conn.prepareStatement(s"select cnts from ${table} where word=?")
                selectStatus.setString(1,tp._1)
                val result = selectStatus.executeQuery()
                var needInsert=true
                while (result.next()){
                  needInsert=false;
                  val Oldvalue = result.getInt(1)
                  val newValue = Oldvalue+tp._2
                  val updateTable = conn.prepareStatement(s"update  ${table} set cnts = ? where word = ?")
                  updateTable.setInt(1,newValue)
                  updateTable.setString(2,tp._1)
                  updateTable.executeUpdate()
                  if (updateTable!=null){
                    updateTable.close()
                  }
                }
                if (needInsert){
                  val insertTable = conn.prepareStatement(s"insert into ${table} values(?,?)")
                  insertTable.setString(1,tp._1)
                  insertTable.setInt(2,tp._2)
                  insertTable.execute()
                  if (insertTable!=null){
                    insertTable.close()
                  }
                }
              })
              if (conn!=null){
                conn.close()
              }

            })

          }
        })
    ssc.start()
    ssc.awaitTermination()
  }

}