spark shell读取hadoop hdfs上文本文件统计wordcount的方法

spark shell读取hadoop hdfs上文本文件统计wordcount的方法

hellospark文件内容如下所示：

[hadoop3@master app]$ hdfs dfs -cat /spark/hellospark
hello spark
hello world
hello spark!

统计结果为：

scala> count.collect();
res0: Array[(String, Int)] = Array((spark!,1), (spark,1), (hello,3), (world,1))

启动spark-shell控制台

[hadoop3@master app]$ spark-shell
2018-08-17 14:37:57 WARN  NativeCodeLoader:62 - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Spark context Web UI available at http://master:4040
Spark context available as 'sc' (master = local[*], app id = local-1534487889053).
Spark session available as 'spark'.
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 2.3.1
      /_/

Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_172)
Type in expressions to have them evaluated.
Type :help for more information.

scala>

从读取hdfs上/spark/hellospark文件，然后map和reduceByKey后，显示count结果

scala> var file=sc.textFile("/spark/hellospark");
scala> val count = file.flatMap(line=>line.split(" ")).map(word=>(word,1)).reduceByKey((a,b)=>a+b)
scala> count.collect();

具体执行过程如下所示：

scala> var file=sc.textFile("/spark/hellospark");
file: org.apache.spark.rdd.RDD[String] = /spark/hellospark MapPartitionsRDD[5] at textFile at <console>:24

scala> val count = file.flatMap(line=>line.split(" ")).map(word=>(word,1)).reduceByKey((a,b)=>a+b)
count: org.apache.spark.rdd.RDD[(String, Int)] = ShuffledRDD[8] at reduceByKey at <console>:25

scala> count.collect();
res0: Array[(String, Int)] = Array((spark!,1), (spark,1), (hello,3), (world,1)) 

scala> 

采用rpc指定的地址访问hdfs上的文件

scala> val file=sc.textFile("hdfs://192.168.10.201:8020/spark/hellospark")
file: org.apache.spark.rdd.RDD[String] = hdfs://192.168.10.201:8020/spark/hellospark MapPartitionsRDD[1] at textFile at <console>:24

scala> val count = file.flatMap(line=>line.split(" ")).map(word=>(word,1)).reduceByKey((a,b)=>a+b)
count: org.apache.spark.rdd.RDD[(String, Int)] = ShuffledRDD[4] at reduceByKey at <console>:25

scala> count.collect();
res0: Array[(String, Int)] = Array((spark!,1), (spark,1), (hello,3), (world,1)) 

scala> 

hdfs-site.xml配置内容为：

<configuration>
    <property>
      <name>dfs.nameservicesname>
      <value>myclustervalue>
    property>
    <property>
      <name>dfs.permissions.enabledname>
      <value>falsevalue>
    property>
    <property>
      <name>dfs.ha.namenodes.myclustername>
      <value>nn1,nn2value>
    property>
    <property>
      <name>dfs.namenode.rpc-address.mycluster.nn1name>
      <value>master:8020value>
    property>
    <property>
      <name>dfs.namenode.rpc-address.mycluster.nn2name>
      <value>slave1:8020value>
    property> 
    <property>
      <name>dfs.namenode.http-address.mycluster.nn1name>
      <value>master:50070value>
    property>
    <property>
      <name>dfs.namenode.http-address.mycluster.nn2name>
      <value>slave1:50070value>
    property>     
    <property>
      <name>dfs.namenode.shared.edits.dirname>
      <value>qjournal://master:8485;slave1:8485/myclustervalue>
    property>
    <property>
      <name>dfs.journalnode.edits.dirname>
      <value>/home/hadoop3/data/journaldata/jnvalue>
    property>
    <property>
      <name>dfs.client.failover.proxy.provider.myclustername>
      <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvidervalue>
    property>
    <property>
      <name>dfs.ha.fencing.methodsname>
      <value>
      sshfence
      shell(/bin/true)
      value>
    property>
    <property>
      <name>dfs.ha.fencing.ssh.private-key-filesname>
      <value>/home/hadoop3/.ssh/id_rsavalue>
    property>
    <property>
      <name>dfs.ha.fencing.ssh.connect-timeoutname>
      <value>10000value>
    property>
    <property>
      <name>dfs.namenode.handler.countname>
      <value>100value>
    property>
    <property>
       <name>dfs.ha.automatic-failover.enabledname>
       <value>truevalue>
     property>
    <property>
         <name>dfs.webhdfs.enabledname>
         <value>truevalue>
    property>  
configuration>

spark Environment

Environment
Runtime Information
Name    Value
Java Version    1.8.0_172 (Oracle Corporation)
Java Home   /home/hadoop3/app/jdk/jre
Scala Version   version 2.11.8
Spark Properties
Name    Value
spark.app.id    local-1534488707608
spark.app.name  Spark shell
spark.driver.host   master
spark.driver.port   46409
spark.executor.id   driver
spark.home  /home/hadoop3/app/spark
spark.jars  
spark.master    local[*]
spark.repl.class.outputDir  /tmp/spark-ed2b6c1c-0472-46b1-a2cb-ab40e341572c/repl-27b565bb-7bcb-4a38-871c-48b0f22749d0
spark.repl.class.uri    spark://master:46409/classes
spark.scheduler.mode    FIFO
spark.sql.catalogImplementation hive
spark.submit.deployMode client
spark.ui.showConsoleProgress    true
System Properties
Name    Value
SPARK_SUBMIT    true
awt.toolkit sun.awt.X11.XToolkit
file.encoding   UTF-8
file.encoding.pkg   sun.io
file.separator  /
java.awt.graphicsenv    sun.awt.X11GraphicsEnvironment
java.awt.printerjob sun.print.PSPrinterJob
java.class.version  52.0
java.endorsed.dirs  /home/hadoop3/app/jdk/jre/lib/endorsed
java.ext.dirs   /home/hadoop3/app/jdk/jre/lib/ext:/usr/java/packages/lib/ext
java.home   /home/hadoop3/app/jdk/jre
java.io.tmpdir  /tmp
java.library.path   /usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
java.runtime.name   Java(TM) SE Runtime Environment
java.runtime.version    1.8.0_172-b11
java.specification.name Java Platform API Specification
java.specification.vendor   Oracle Corporation
java.specification.version  1.8
java.vendor Oracle Corporation
java.vendor.url http://java.oracle.com/
java.vendor.url.bug http://bugreport.sun.com/bugreport/
java.version    1.8.0_172
java.vm.info    mixed mode
java.vm.name    Java HotSpot(TM) 64-Bit Server VM
java.vm.specification.name  Java Virtual Machine Specification
java.vm.specification.vendor    Oracle Corporation
java.vm.specification.version   1.8
java.vm.vendor  Oracle Corporation
java.vm.version 25.172-b11
line.separator  
os.arch amd64
os.name Linux
os.version  3.10.0-514.el7.x86_64
path.separator  :
scala.usejavacp true
sun.arch.data.model 64
sun.boot.class.path /home/hadoop3/app/jdk/jre/lib/resources.jar:/home/hadoop3/app/jdk/jre/lib/rt.jar:/home/hadoop3/app/jdk/jre/lib/sunrsasign.jar:/home/hadoop3/app/jdk/jre/lib/jsse.jar:/home/hadoop3/app/jdk/jre/lib/jce.jar:/home/hadoop3/app/jdk/jre/lib/charsets.jar:/home/hadoop3/app/jdk/jre/lib/jfr.jar:/home/hadoop3/app/jdk/jre/classes
sun.boot.library.path   /home/hadoop3/app/jdk/jre/lib/amd64
sun.cpu.endian  little
sun.cpu.isalist 
sun.io.unicode.encoding UnicodeLittle
sun.java.command    org.apache.spark.deploy.SparkSubmit --class org.apache.spark.repl.Main --name Spark shell spark-shell
sun.java.launcher   SUN_STANDARD
sun.jnu.encoding    UTF-8
sun.management.compiler HotSpot 64-Bit Tiered Compilers
sun.nio.ch.bugLevel 
sun.os.patch.level  unknown
user.country    CN
user.dir    /home/hadoop3/app
user.home   /home/hadoop3
user.language   zh
user.name   hadoop3
user.timezone   Asia/Shanghai
Classpath Entries
Resource    Source
/home/hadoop3/app/hadoop/etc/hadoop/    System Classpath
/home/hadoop3/app/spark/conf/   System Classpath
/home/hadoop3/app/spark/jars/JavaEWAH-0.3.2.jar System Classpath
/home/hadoop3/app/spark/jars/RoaringBitmap-0.5.11.jar   System Classpath
/home/hadoop3/app/spark/jars/ST4-4.0.4.jar  System Classpath
/home/hadoop3/app/spark/jars/activation-1.1.1.jar   System Classpath
/home/hadoop3/app/spark/jars/aircompressor-0.8.jar  System Classpath
/home/hadoop3/app/spark/jars/antlr-2.7.7.jar    System Classpath
/home/hadoop3/app/spark/jars/antlr-runtime-3.4.jar  System Classpath
/home/hadoop3/app/spark/jars/antlr4-runtime-4.7.jar System Classpath
/home/hadoop3/app/spark/jars/aopalliance-1.0.jar    System Classpath
/home/hadoop3/app/spark/jars/aopalliance-repackaged-2.4.0-b34.jar   System Classpath
/home/hadoop3/app/spark/jars/apache-log4j-extras-1.2.17.jar System Classpath
/home/hadoop3/app/spark/jars/apacheds-i18n-2.0.0-M15.jar    System Classpath
/home/hadoop3/app/spark/jars/apacheds-kerberos-codec-2.0.0-M15.jar  System Classpath
/home/hadoop3/app/spark/jars/api-asn1-api-1.0.0-M20.jar System Classpath
/home/hadoop3/app/spark/jars/api-util-1.0.0-M20.jar System Classpath
/home/hadoop3/app/spark/jars/arpack_combined_all-0.1.jar    System Classpath
/home/hadoop3/app/spark/jars/arrow-format-0.8.0.jar System Classpath
/home/hadoop3/app/spark/jars/arrow-memory-0.8.0.jar System Classpath
/home/hadoop3/app/spark/jars/arrow-vector-0.8.0.jar System Classpath
/home/hadoop3/app/spark/jars/automaton-1.11-8.jar   System Classpath
/home/hadoop3/app/spark/jars/avro-1.7.7.jar System Classpath
/home/hadoop3/app/spark/jars/avro-ipc-1.7.7.jar System Classpath
/home/hadoop3/app/spark/jars/avro-mapred-1.7.7-hadoop2.jar  System Classpath
/home/hadoop3/app/spark/jars/base64-2.3.8.jar   System Classpath
/home/hadoop3/app/spark/jars/bcprov-jdk15on-1.58.jar    System Classpath
/home/hadoop3/app/spark/jars/bonecp-0.8.0.RELEASE.jar   System Classpath
/home/hadoop3/app/spark/jars/breeze-macros_2.11-0.13.2.jar  System Classpath
/home/hadoop3/app/spark/jars/breeze_2.11-0.13.2.jar System Classpath
/home/hadoop3/app/spark/jars/calcite-avatica-1.2.0-incubating.jar   System Classpath
/home/hadoop3/app/spark/jars/calcite-core-1.2.0-incubating.jar  System Classpath
/home/hadoop3/app/spark/jars/calcite-linq4j-1.2.0-incubating.jar    System Classpath
/home/hadoop3/app/spark/jars/chill-java-0.8.4.jar   System Classpath
/home/hadoop3/app/spark/jars/chill_2.11-0.8.4.jar   System Classpath
/home/hadoop3/app/spark/jars/commons-beanutils-1.7.0.jar    System Classpath
/home/hadoop3/app/spark/jars/commons-beanutils-core-1.8.0.jar   System Classpath
/home/hadoop3/app/spark/jars/commons-cli-1.2.jar    System Classpath
/home/hadoop3/app/spark/jars/commons-codec-1.10.jar System Classpath
/home/hadoop3/app/spark/jars/commons-collections-3.2.2.jar  System Classpath
/home/hadoop3/app/spark/jars/commons-compiler-3.0.8.jar System Classpath
/home/hadoop3/app/spark/jars/commons-compress-1.4.1.jar System Classpath
/home/hadoop3/app/spark/jars/commons-configuration-1.6.jar  System Classpath
/home/hadoop3/app/spark/jars/commons-crypto-1.0.0.jar   System Classpath
/home/hadoop3/app/spark/jars/commons-dbcp-1.4.jar   System Classpath
/home/hadoop3/app/spark/jars/commons-digester-1.8.jar   System Classpath
/home/hadoop3/app/spark/jars/commons-httpclient-3.1.jar System Classpath
/home/hadoop3/app/spark/jars/commons-io-2.4.jar System Classpath
/home/hadoop3/app/spark/jars/commons-lang-2.6.jar   System Classpath
/home/hadoop3/app/spark/jars/commons-lang3-3.5.jar  System Classpath
/home/hadoop3/app/spark/jars/commons-logging-1.1.3.jar  System Classpath
/home/hadoop3/app/spark/jars/commons-math3-3.4.1.jar    System Classpath
/home/hadoop3/app/spark/jars/commons-net-2.2.jar    System Classpath
/home/hadoop3/app/spark/jars/commons-pool-1.5.4.jar System Classpath
/home/hadoop3/app/spark/jars/compress-lzf-1.0.3.jar System Classpath
/home/hadoop3/app/spark/jars/core-1.1.2.jar System Classpath
/home/hadoop3/app/spark/jars/curator-client-2.7.1.jar   System Classpath
/home/hadoop3/app/spark/jars/curator-framework-2.7.1.jar    System Classpath
/home/hadoop3/app/spark/jars/curator-recipes-2.7.1.jar  System Classpath
/home/hadoop3/app/spark/jars/datanucleus-api-jdo-3.2.6.jar  System Classpath
/home/hadoop3/app/spark/jars/datanucleus-core-3.2.10.jar    System Classpath
/home/hadoop3/app/spark/jars/datanucleus-rdbms-3.2.9.jar    System Classpath
/home/hadoop3/app/spark/jars/derby-10.12.1.1.jar    System Classpath
/home/hadoop3/app/spark/jars/eigenbase-properties-1.1.5.jar System Classpath
/home/hadoop3/app/spark/jars/flatbuffers-1.2.0-3f79e055.jar System Classpath
/home/hadoop3/app/spark/jars/generex-1.0.1.jar  System Classpath
/home/hadoop3/app/spark/jars/gson-2.2.4.jar System Classpath
/home/hadoop3/app/spark/jars/guava-14.0.1.jar   System Classpath
/home/hadoop3/app/spark/jars/guice-3.0.jar  System Classpath
/home/hadoop3/app/spark/jars/guice-servlet-3.0.jar  System Classpath
/home/hadoop3/app/spark/jars/hadoop-annotations-2.7.3.jar   System Classpath
/home/hadoop3/app/spark/jars/hadoop-auth-2.7.3.jar  System Classpath
/home/hadoop3/app/spark/jars/hadoop-client-2.7.3.jar    System Classpath
/home/hadoop3/app/spark/jars/hadoop-common-2.7.3.jar    System Classpath
/home/hadoop3/app/spark/jars/hadoop-hdfs-2.7.3.jar  System Classpath
/home/hadoop3/app/spark/jars/hadoop-mapreduce-client-app-2.7.3.jar  System Classpath
/home/hadoop3/app/spark/jars/hadoop-mapreduce-client-common-2.7.3.jar   System Classpath
/home/hadoop3/app/spark/jars/hadoop-mapreduce-client-core-2.7.3.jar System Classpath
/home/hadoop3/app/spark/jars/hadoop-mapreduce-client-jobclient-2.7.3.jar    System Classpath
/home/hadoop3/app/spark/jars/hadoop-mapreduce-client-shuffle-2.7.3.jar  System Classpath
/home/hadoop3/app/spark/jars/hadoop-yarn-api-2.7.3.jar  System Classpath
/home/hadoop3/app/spark/jars/hadoop-yarn-client-2.7.3.jar   System Classpath
/home/hadoop3/app/spark/jars/hadoop-yarn-common-2.7.3.jar   System Classpath
/home/hadoop3/app/spark/jars/hadoop-yarn-server-common-2.7.3.jar    System Classpath
/home/hadoop3/app/spark/jars/hadoop-yarn-server-web-proxy-2.7.3.jar System Classpath
/home/hadoop3/app/spark/jars/hive-beeline-1.2.1.spark2.jar  System Classpath
/home/hadoop3/app/spark/jars/hive-cli-1.2.1.spark2.jar  System Classpath
/home/hadoop3/app/spark/jars/hive-exec-1.2.1.spark2.jar System Classpath
/home/hadoop3/app/spark/jars/hive-jdbc-1.2.1.spark2.jar System Classpath
/home/hadoop3/app/spark/jars/hive-metastore-1.2.1.spark2.jar    System Classpath
/home/hadoop3/app/spark/jars/hk2-api-2.4.0-b34.jar  System Classpath
/home/hadoop3/app/spark/jars/hk2-locator-2.4.0-b34.jar  System Classpath
/home/hadoop3/app/spark/jars/hk2-utils-2.4.0-b34.jar    System Classpath
/home/hadoop3/app/spark/jars/hppc-0.7.2.jar System Classpath
/home/hadoop3/app/spark/jars/htrace-core-3.1.0-incubating.jar   System Classpath
/home/hadoop3/app/spark/jars/httpclient-4.5.4.jar   System Classpath
/home/hadoop3/app/spark/jars/httpcore-4.4.8.jar System Classpath
/home/hadoop3/app/spark/jars/ivy-2.4.0.jar  System Classpath
/home/hadoop3/app/spark/jars/jackson-annotations-2.6.7.jar  System Classpath
/home/hadoop3/app/spark/jars/jackson-core-2.6.7.jar System Classpath
/home/hadoop3/app/spark/jars/jackson-core-asl-1.9.13.jar    System Classpath
/home/hadoop3/app/spark/jars/jackson-databind-2.6.7.1.jar   System Classpath
/home/hadoop3/app/spark/jars/jackson-dataformat-yaml-2.6.7.jar  System Classpath
/home/hadoop3/app/spark/jars/jackson-jaxrs-1.9.13.jar   System Classpath
/home/hadoop3/app/spark/jars/jackson-mapper-asl-1.9.13.jar  System Classpath
/home/hadoop3/app/spark/jars/jackson-module-jaxb-annotations-2.6.7.jar  System Classpath
/home/hadoop3/app/spark/jars/jackson-module-paranamer-2.7.9.jar System Classpath
/home/hadoop3/app/spark/jars/jackson-module-scala_2.11-2.6.7.1.jar  System Classpath
/home/hadoop3/app/spark/jars/jackson-xc-1.9.13.jar  System Classpath
/home/hadoop3/app/spark/jars/janino-3.0.8.jar   System Classpath
/home/hadoop3/app/spark/jars/java-xmlbuilder-1.1.jar    System Classpath
/home/hadoop3/app/spark/jars/javassist-3.18.1-GA.jar    System Classpath
/home/hadoop3/app/spark/jars/javax.annotation-api-1.2.jar   System Classpath
/home/hadoop3/app/spark/jars/javax.inject-1.jar System Classpath
/home/hadoop3/app/spark/jars/javax.inject-2.4.0-b34.jar System Classpath
/home/hadoop3/app/spark/jars/javax.servlet-api-3.1.0.jar    System Classpath
/home/hadoop3/app/spark/jars/javax.ws.rs-api-2.0.1.jar  System Classpath
/home/hadoop3/app/spark/jars/javolution-5.5.1.jar   System Classpath
/home/hadoop3/app/spark/jars/jaxb-api-2.2.2.jar System Classpath
/home/hadoop3/app/spark/jars/jcl-over-slf4j-1.7.16.jar  System Classpath
/home/hadoop3/app/spark/jars/jdo-api-3.0.1.jar  System Classpath
/home/hadoop3/app/spark/jars/jersey-client-2.22.2.jar   System Classpath
/home/hadoop3/app/spark/jars/jersey-common-2.22.2.jar   System Classpath
/home/hadoop3/app/spark/jars/jersey-container-servlet-2.22.2.jar    System Classpath
/home/hadoop3/app/spark/jars/jersey-container-servlet-core-2.22.2.jar   System Classpath
/home/hadoop3/app/spark/jars/jersey-guava-2.22.2.jar    System Classpath
/home/hadoop3/app/spark/jars/jersey-media-jaxb-2.22.2.jar   System Classpath
/home/hadoop3/app/spark/jars/jersey-server-2.22.2.jar   System Classpath
/home/hadoop3/app/spark/jars/jets3t-0.9.4.jar   System Classpath
/home/hadoop3/app/spark/jars/jetty-6.1.26.jar   System Classpath
/home/hadoop3/app/spark/jars/jetty-util-6.1.26.jar  System Classpath
/home/hadoop3/app/spark/jars/jline-2.12.1.jar   System Classpath
/home/hadoop3/app/spark/jars/joda-time-2.9.3.jar    System Classpath
/home/hadoop3/app/spark/jars/jodd-core-3.5.2.jar    System Classpath
/home/hadoop3/app/spark/jars/jpam-1.1.jar   System Classpath
/home/hadoop3/app/spark/jars/json4s-ast_2.11-3.2.11.jar System Classpath
/home/hadoop3/app/spark/jars/json4s-core_2.11-3.2.11.jar    System Classpath
/home/hadoop3/app/spark/jars/json4s-jackson_2.11-3.2.11.jar System Classpath
/home/hadoop3/app/spark/jars/jsp-api-2.1.jar    System Classpath
/home/hadoop3/app/spark/jars/jsr305-1.3.9.jar   System Classpath
/home/hadoop3/app/spark/jars/jta-1.1.jar    System Classpath
/home/hadoop3/app/spark/jars/jtransforms-2.4.0.jar  System Classpath
/home/hadoop3/app/spark/jars/jul-to-slf4j-1.7.16.jar    System Classpath
/home/hadoop3/app/spark/jars/kryo-shaded-3.0.3.jar  System Classpath
/home/hadoop3/app/spark/jars/kubernetes-client-3.0.0.jar    System Classpath
/home/hadoop3/app/spark/jars/kubernetes-model-2.0.0.jar System Classpath
/home/hadoop3/app/spark/jars/leveldbjni-all-1.8.jar System Classpath
/home/hadoop3/app/spark/jars/libfb303-0.9.3.jar System Classpath
/home/hadoop3/app/spark/jars/libthrift-0.9.3.jar    System Classpath
/home/hadoop3/app/spark/jars/log4j-1.2.17.jar   System Classpath
/home/hadoop3/app/spark/jars/logging-interceptor-3.8.1.jar  System Classpath
/home/hadoop3/app/spark/jars/lz4-java-1.4.0.jar System Classpath
/home/hadoop3/app/spark/jars/machinist_2.11-0.6.1.jar   System Classpath
/home/hadoop3/app/spark/jars/macro-compat_2.11-1.1.1.jar    System Classpath
/home/hadoop3/app/spark/jars/mesos-1.4.0-shaded-protobuf.jar    System Classpath
/home/hadoop3/app/spark/jars/metrics-core-3.1.5.jar System Classpath
/home/hadoop3/app/spark/jars/metrics-graphite-3.1.5.jar System Classpath
/home/hadoop3/app/spark/jars/metrics-json-3.1.5.jar System Classpath
/home/hadoop3/app/spark/jars/metrics-jvm-3.1.5.jar  System Classpath
/home/hadoop3/app/spark/jars/minlog-1.3.0.jar   System Classpath
/home/hadoop3/app/spark/jars/netty-3.9.9.Final.jar  System Classpath
/home/hadoop3/app/spark/jars/netty-all-4.1.17.Final.jar System Classpath
/home/hadoop3/app/spark/jars/objenesis-2.1.jar  System Classpath
/home/hadoop3/app/spark/jars/okhttp-3.8.1.jar   System Classpath
/home/hadoop3/app/spark/jars/okio-1.13.0.jar    System Classpath
/home/hadoop3/app/spark/jars/opencsv-2.3.jar    System Classpath
/home/hadoop3/app/spark/jars/orc-core-1.4.4-nohive.jar  System Classpath
/home/hadoop3/app/spark/jars/orc-mapreduce-1.4.4-nohive.jar System Classpath
/home/hadoop3/app/spark/jars/oro-2.0.8.jar  System Classpath
/home/hadoop3/app/spark/jars/osgi-resource-locator-1.0.1.jar    System Classpath
/home/hadoop3/app/spark/jars/paranamer-2.8.jar  System Classpath
/home/hadoop3/app/spark/jars/parquet-column-1.8.3.jar   System Classpath
/home/hadoop3/app/spark/jars/parquet-common-1.8.3.jar   System Classpath
/home/hadoop3/app/spark/jars/parquet-encoding-1.8.3.jar System Classpath
/home/hadoop3/app/spark/jars/parquet-format-2.3.1.jar   System Classpath
/home/hadoop3/app/spark/jars/parquet-hadoop-1.8.3.jar   System Classpath
/home/hadoop3/app/spark/jars/parquet-hadoop-bundle-1.6.0.jar    System Classpath
/home/hadoop3/app/spark/jars/parquet-jackson-1.8.3.jar  System Classpath
/home/hadoop3/app/spark/jars/protobuf-java-2.5.0.jar    System Classpath
/home/hadoop3/app/spark/jars/py4j-0.10.7.jar    System Classpath
/home/hadoop3/app/spark/jars/pyrolite-4.13.jar  System Classpath
/home/hadoop3/app/spark/jars/scala-compiler-2.11.8.jar  System Classpath
/home/hadoop3/app/spark/jars/scala-library-2.11.8.jar   System Classpath
/home/hadoop3/app/spark/jars/scala-parser-combinators_2.11-1.0.4.jar    System Classpath
/home/hadoop3/app/spark/jars/scala-reflect-2.11.8.jar   System Classpath
/home/hadoop3/app/spark/jars/scala-xml_2.11-1.0.5.jar   System Classpath
/home/hadoop3/app/spark/jars/scalap-2.11.8.jar  System Classpath
/home/hadoop3/app/spark/jars/shapeless_2.11-2.3.2.jar   System Classpath
/home/hadoop3/app/spark/jars/slf4j-api-1.7.16.jar   System Classpath
/home/hadoop3/app/spark/jars/slf4j-log4j12-1.7.16.jar   System Classpath
/home/hadoop3/app/spark/jars/snakeyaml-1.15.jar System Classpath
/home/hadoop3/app/spark/jars/snappy-0.2.jar System Classpath
/home/hadoop3/app/spark/jars/snappy-java-1.1.2.6.jar    System Classpath
/home/hadoop3/app/spark/jars/spark-catalyst_2.11-2.3.1.jar  System Classpath
/home/hadoop3/app/spark/jars/spark-core_2.11-2.3.1.jar  System Classpath
/home/hadoop3/app/spark/jars/spark-graphx_2.11-2.3.1.jar    System Classpath
/home/hadoop3/app/spark/jars/spark-hive-thriftserver_2.11-2.3.1.jar System Classpath
/home/hadoop3/app/spark/jars/spark-hive_2.11-2.3.1.jar  System Classpath
/home/hadoop3/app/spark/jars/spark-kubernetes_2.11-2.3.1.jar    System Classpath
/home/hadoop3/app/spark/jars/spark-kvstore_2.11-2.3.1.jar   System Classpath
/home/hadoop3/app/spark/jars/spark-launcher_2.11-2.3.1.jar  System Classpath
/home/hadoop3/app/spark/jars/spark-mesos_2.11-2.3.1.jar System Classpath
/home/hadoop3/app/spark/jars/spark-mllib-local_2.11-2.3.1.jar   System Classpath
/home/hadoop3/app/spark/jars/spark-mllib_2.11-2.3.1.jar System Classpath
/home/hadoop3/app/spark/jars/spark-network-common_2.11-2.3.1.jar    System Classpath
/home/hadoop3/app/spark/jars/spark-network-shuffle_2.11-2.3.1.jar   System Classpath
/home/hadoop3/app/spark/jars/spark-repl_2.11-2.3.1.jar  System Classpath
/home/hadoop3/app/spark/jars/spark-sketch_2.11-2.3.1.jar    System Classpath
/home/hadoop3/app/spark/jars/spark-sql_2.11-2.3.1.jar   System Classpath
/home/hadoop3/app/spark/jars/spark-streaming_2.11-2.3.1.jar System Classpath
/home/hadoop3/app/spark/jars/spark-tags_2.11-2.3.1.jar  System Classpath
/home/hadoop3/app/spark/jars/spark-unsafe_2.11-2.3.1.jar    System Classpath
/home/hadoop3/app/spark/jars/spark-yarn_2.11-2.3.1.jar  System Classpath
/home/hadoop3/app/spark/jars/spire-macros_2.11-0.13.0.jar   System Classpath
/home/hadoop3/app/spark/jars/spire_2.11-0.13.0.jar  System Classpath
/home/hadoop3/app/spark/jars/stax-api-1.0-2.jar System Classpath
/home/hadoop3/app/spark/jars/stax-api-1.0.1.jar System Classpath
/home/hadoop3/app/spark/jars/stream-2.7.0.jar   System Classpath
/home/hadoop3/app/spark/jars/stringtemplate-3.2.1.jar   System Classpath
/home/hadoop3/app/spark/jars/super-csv-2.2.0.jar    System Classpath
/home/hadoop3/app/spark/jars/univocity-parsers-2.5.9.jar    System Classpath
/home/hadoop3/app/spark/jars/validation-api-1.1.0.Final.jar System Classpath
/home/hadoop3/app/spark/jars/xbean-asm5-shaded-4.4.jar  System Classpath
/home/hadoop3/app/spark/jars/xercesImpl-2.9.1.jar   System Classpath
/home/hadoop3/app/spark/jars/xmlenc-0.52.jar    System Classpath
/home/hadoop3/app/spark/jars/xz-1.0.jar System Classpath
/home/hadoop3/app/spark/jars/zjsonpatch-0.3.0.jar   System Classpath
/home/hadoop3/app/spark/jars/zookeeper-3.4.6.jar    System Classpath
/home/hadoop3/app/spark/jars/zstd-jni-1.3.2-2.jar   System Classpath