spark-IDEA之SBT打包

SBT打包代码与依赖：

这里我使用的是SBT进行的打包:

 

1、创建assembly.sbt并在里面写:

addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")

 

 

2、Build.sbt中写:

import AssemblyKeys._

name := "saprk-sbt"

version := "1.0"

scalaVersion := "2.10.4"

libraryDependencies ++= Seq(
  //spark依赖
  "org.apache.spark" % "spark-core_2.10" % "1.6.1" % "provided",
  "com.alibaba" % "fastjson" % "1.2.24",
  /*
   * kafka
   * */
  "org.apache.spark" % "spark-streaming_2.10" % "1.6.1" % "provided",
  "org.apache.spark" % "spark-streaming-kafka_2.10" % "1.6.1" % "provided",
  /*
   * commions io
   * */
  "commons-io" % "commons-io" % "2.4",
  /*
   * joda
   * */
  "joda-time" % "joda-time" % "2.3"
)
//这条语句打开了assembly的插件功能
assemblySettings

//执行assembly的时候忽略测试
test in assembly :={}

//指定类的名字
mainClass in assembly := Some("auto.DateManual")

assemblyOption in packageDependency ~= {
  _.copy(appendContentHash = true)
}
/**
  * MergeStrategy.first:默认取得第一个匹配项
  * MergeStrategy.last:默认取得最后一个匹配项
  * MergeStrategy.discard:近丢弃匹配的文件
  * */
//解决依赖重复的问题
mergeStrategy in assembly := {
  //  case x if x.startsWith("META-INF") => MergeStrategy.discard
  //  case x if x.endsWith(".class") => MergeStrategy.discard
  //    case x if x.contains("slf4j-api") => MergeStrategy.last
  //    case x if x.contains("org/cyberneko/html") => MergeStrategy.first
  //如果后缀是.properties的文件，合并策略采用（MergeStrategy.first）第一个出现的文件
  case PathList(ps@_*) if ps.last endsWith ".properties" => MergeStrategy.first
  case PathList(ps@_*) if ps.last endsWith "Absent.class" => MergeStrategy.first
  case PathList("com", "esotericsoftware", xs@_*) => MergeStrategy.first
  case x =>
    val oldStrategy = (mergeStrategy in assembly).value
    oldStrategy(x)
}
//定义jar包的名字
jarName in assembly := "sbt-solr-assembly.jar"

//把scala本身排除在Jar中，因为spark已经包含了scala
assemblyOption in assembly :=

(assemblyOption in assembly).value.copy(includeScala = false)

3、

 

-----

中运行sbt 然后运行clean,compile,assembly(打包命令)