Maven 配置Spark + hadoop

        工程基于Spark  2.0.0 + hadoop 2.8.0 构建,数据裤使用MongoDB,以下为配置详情.如果遇到版本冲突,用以下命令查看maven 依赖关系,排除错误依赖版本,配置所缺依赖版本

一   查看依赖

mvn生成依赖解析树,用于查看版本冲突,哪个版本被使用
mvn dependency:tree
mvn dependency:tree >>dependency.log

二  pom.xml配置

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <modelVersion>4.0.0modelVersion>
  <groupId>工程目录groupId>
  <artifactId>工程名artifactId>
  <version>1.0-SNAPSHOTversion>
  <inceptionYear>2008inceptionYear>


  <properties>
    <scala.version>2.12.3scala.version>
    <spark.artifactID.suffix>2.11spark.artifactID.suffix>
    <spark.version>2.2.0spark.version>
    <hadoop.version>2.7.4hadoop.version>
 
  properties>

  <repositories>
    <repository>
      <id>scala-tools.orgid>
      <name>Scala-Tools Maven2 Repositoryname>
      <url>http://scala-tools.org/repo-releasesurl>
    repository>
  repositories>

  <pluginRepositories>
    <pluginRepository>
      <id>scala-tools.orgid>
      <name>Scala-Tools Maven2 Repositoryname>
      <url>http://scala-tools.org/repo-releasesurl>
    pluginRepository>
  pluginRepositories>

  <dependencies>


            

    


    
      -->

    
    <dependency>
      <groupId>junitgroupId>
      <artifactId>junitartifactId>
      <version>4.4version>
      <scope>testscope>
    dependency>
    <dependency>
      <groupId>org.specsgroupId>
      <artifactId>specsartifactId>
      <version>1.2.5version>
      <scope>testscope>
    dependency>





    
    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-core_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>
    dependency>

    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-streaming_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>

    dependency>


    
    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-sql_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>

    dependency>

    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-hive_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>

    dependency>
    <dependency>
      <groupId>org.apache.sparkgroupId>
      <artifactId>spark-mllib_${spark.artifactID.suffix}artifactId>
      <version>${spark.version}version>

    dependency>


    

    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-commonartifactId>
      <version>${hadoop.version}version>
    dependency>
    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-clientartifactId>
      <version>${hadoop.version}version>
    dependency>


    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-hdfsartifactId>
      <version>${hadoop.version}version>
    dependency>




    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-mapreduce-client-coreartifactId>
      <version>${hadoop.version}version>
      <exclusions>
        <exclusion>
          <artifactId>org.apache.hadoopartifactId>
          <groupId>hadoop-coregroupId>
        exclusion>
      exclusions>
    dependency>

    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-annotationsartifactId>
      <version>${hadoop.version}version>
    dependency>

    <dependency>
      <groupId>org.apache.hadoopgroupId>
      <artifactId>hadoop-miniclusterartifactId>
      <version>${hadoop.version}version>
      <scope>testscope>
    dependency>


    

 

    <dependency>
      <groupId>org.mongodbgroupId>
      <artifactId>mongo-java-driverartifactId>
      <version>3.4.2version>
    dependency>



    <dependency>
      <groupId>org.mongodb.sparkgroupId>
      <artifactId>mongo-spark-connector_2.11artifactId>
      <version>2.2.0version>
    dependency>
 


    <dependency>
      <groupId>com.alibabagroupId>
      <artifactId>fastjsonartifactId>
      <version>1.2.44version>
    dependency>


  dependencies>

  <build>
    <sourceDirectory>src/main/scalasourceDirectory>
    <testSourceDirectory>src/test/scalatestSourceDirectory>
    <plugins>
      <plugin>
        <groupId>org.scala-toolsgroupId>
        <artifactId>maven-scala-pluginartifactId>
        <executions>
          <execution>
            <goals>
              <goal>compilegoal>
              <goal>testCompilegoal>
            goals>
          execution>
        executions>
        <configuration>
          <scalaVersion>${scala.version}scalaVersion>
          <args>
            <arg>-target:jvm-1.5arg>
          args>
        configuration>
      plugin>
      <plugin>
        <groupId>org.apache.maven.pluginsgroupId>
        <artifactId>maven-eclipse-pluginartifactId>
        <configuration>
          <downloadSources>truedownloadSources>
          <buildcommands>
            <buildcommand>ch.epfl.lamp.sdt.core.scalabuilderbuildcommand>
          buildcommands>
          <additionalProjectnatures>
            <projectnature>ch.epfl.lamp.sdt.core.scalanatureprojectnature>
          additionalProjectnatures>
          <classpathContainers>
            <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINERclasspathContainer>
            <classpathContainer>ch.epfl.lamp.sdt.launching.SCALA_CONTAINERclasspathContainer>
          classpathContainers>
        configuration>
      plugin>



      
      

<plugin>
  <artifactId>maven-compiler-pluginartifactId>
  <configuration>
    <source>1.7source>
    <target>1.7target>
    <encoding>${project.build.sourceEncoding}encoding>
  configuration>
plugin>
<plugin>
  <artifactId>maven-assembly-pluginartifactId>
  <configuration>
    <descriptorRefs>
      <descriptorRef>jar-with-dependenciesdescriptorRef>
    descriptorRefs>
    
  configuration>
  <executions>
    <execution>
      <id>make-assemblyid> 
      <phase>packagephase> 
      <goals>
        <goal>singlegoal>
      goals>
    execution>
  executions>
plugin>


 
<plugin>
  <artifactId>maven-shade-pluginartifactId>
  <version>2.4.3version>
  <executions>
    <execution>
      <phase>packagephase>
      <goals>
        <goal>shadegoal>
      goals>
      <configuration>
        <filters>
          <filter>
            <artifact>*:*artifact>
            <excludes>
              <exclude>META-INF/*.SFexclude>
              <exclude>META-INF/*.DSAexclude>
              <exclude>META-INF/*.RSAexclude>
            excludes>
          filter>
        filters>
      configuration>
    execution>
  executions>
plugin>

plugins > build > < reporting > < plugins > < plugin > < groupId >org.scala-tools groupId > < artifactId >maven-scala-plugin artifactId > < configuration > < scalaVersion >${scala.version} scalaVersion > configuration > plugin > plugins > reporting > project >