SpringBoot + Spark

Scala开发，这里只是Spark local模式的简单示例，工作也没有需求 http请求去执行Spark yarn，所以暂时没有去管。

 pom.xml

需要注意的是去除掉 SpringBoot 默认的日志引擎。

    
        UTF-8
        1.7.28
        2.12.1
        2.11.12
        2.4.0
        2.1.1
        5.1.47
    

    
        org.springframework.boot
        spring-boot-starter-parent
        2.1.7.RELEASE
        
    

    
        
            org.slf4j
            slf4j-nop
            ${slf4j.version}
        

        
        
            org.scala-lang
            scala-library
            ${scala.version}
        

        
        
            org.apache.spark
            spark-core_2.11
            ${spark.version}
            
                
                    org.slf4j
                    slf4j-log4j12
                
                
                    log4j
                    log4j
                
            
            compile
        
        
            org.apache.spark
            spark-sql_2.11
            ${spark.version}
        
        
            org.apache.spark
            spark-hive_2.11
            ${spark.version}
        
        
            org.apache.spark
            spark-yarn_2.11
            ${spark.version}
        


        
            mysql
            mysql-connector-java
        

        
            org.codehaus.janino
            janino
            3.0.8
        

        
        
            org.springframework.boot
            spring-boot-starter
            
                
                    org.springframework.boot
                    spring-boot-starter-logging
                
            
        
        
            org.springframework.boot
            spring-boot-starter-aop
        
        
            org.springframework.boot
            spring-boot-starter-web
        
        
            org.springframework.boot
            spring-boot-configuration-processor
            true
        

        
            commons-codec
            commons-codec
            1.9
        
    

    
        
            
                org.apache.maven.plugins
                maven-compiler-plugin
                3.6.1
                
                    1.8
                    1.8
                
            
            
                org.springframework.boot
                spring-boot-maven-plugin
                
                    true
                
            
        
    

 

SparkConfig

这里可以在application.properties/ yml 中配置参数，这里测试就简单写死。

import java.io.File

import org.apache.spark.sql.SparkSession
import org.springframework.boot.context.properties.ConfigurationProperties
import org.springframework.stereotype.Component

/**
  * create by LiuJinHe 2019/9/25
  */
@Component
@ConfigurationProperties(prefix = "spark-config")
class SparkConfig {
  private val warehouseLocation = new File("spark-warehouse").getAbsolutePath
  private val appName:String = "SparkJob"
  private val master:String = "local"

  def initSparkSession: SparkSession = SparkSession.builder
    .appName(appName)
    .master(master)
    .config("spark.sql.warehouse.dir", warehouseLocation)
    .enableHiveSupport()
    .getOrCreate()
}

 

SpringBootApplication。

import org.springframework.boot.SpringApplication
import org.springframework.boot.autoconfigure.{EnableAutoConfiguration, SpringBootApplication}
import org.springframework.context.annotation.ComponentScan

/**
  * SpringBoot 入口
  *
  * create by LiuJinHe 2019/9/25
  */
@EnableAutoConfiguration
@ComponentScan
@SpringBootApplication
class CrowdPackageApp

object CrowdPackageApp extends App {
  SpringApplication.run(classOf[CrowdPackageApp])
}

 

RestController

这里直接把SparkJob写成了object，没有通过注解@Autowired 来实例化。

import com.sm.service.CrowdService
import org.springframework.web.bind.annotation.{CrossOrigin, RequestMapping, RequestMethod, RequestParam, 
ResponseBody, RestController}

/**
  * 请求地址
  *
  * create by LiuJinHe 2019/9/25
  */
@RestController
@CrossOrigin
class CrowdController {

  /**
    * 请求
    */
  @RequestMapping(value = Array("/crowd"), method = Array(RequestMethod.GET))
  @ResponseBody
    def exportCrowd(@RequestParam db: String, @RequestParam table: String): String  = {
    SparkJob.getGameId(db, table)
  }
}

 

SparkJob 

spark任务执行类。

import com.sm.config.SparkConfig
import org.apache.spark.sql.SparkSession
import org.springframework.beans.factory.annotation.Autowired

/**
  * create by LiuJinHe 2019/9/25
  */
object SparkJob {

  @Autowired
  var sparkConf: SparkConfig = new SparkConfig
  var spark:SparkSession = sparkConf.initSparkSession

  def getGameId(db: String,table:String):String = {
//    val url = s"jdbc:mysql://localhost:3306/${db}?characterEncoding=UTF-8&serverTimezone=UTC&user=root&password=863863"
//    val prop = new Properties()
//    val df = spark.read.jdbc(url, $table, prop)
//    df.createOrReplaceTempView("tmpTable")
//    val dataFrame = spark.sql("select * from tmpTable")
//    dataFrame.show().rdd.toString

    spark.sql(s"select * from $db.$table limit 10").show().toString
  }
}