spark与java的联系
spark依赖于java和python，
（一）看了下python的版本python -v，没想到反应这么大，进入了python命令模式。
[root@localhost spark]# python -v
# installing zipimport hook
import zipimport # builtin
# installed zipimport hook
# /usr/lib/python2.6/site.pyc matches /usr/lib/python2.6/site.py
import site # precompiled from /usr/lib/python2.6/site.pyc
# /usr/lib/python2.6/os.pyc matches /usr/lib/python2.6/os.py
import os # precompiled from /usr/lib/python2.6/os.pyc
import errno # builtin
import posix # builtin
# /usr/lib/python2.6/posixpath.pyc matches /usr/lib/python2.6/posixpath.py
import posixpath # precompiled from /usr/lib/python2.6/posixpath.pyc
# /usr/lib/python2.6/stat.pyc matches /usr/lib/python2.6/stat.py
import stat # precompiled from /usr/lib/python2.6/stat.pyc
# /usr/lib/python2.6/genericpath.pyc matches /usr/lib/python2.6/genericpath.py
import genericpath # precompiled from /usr/lib/python2.6/genericpath.pyc
# /usr/lib/python2.6/warnings.pyc matches /usr/lib/python2.6/warnings.py
import warnings # precompiled from /usr/lib/python2.6/warnings.pyc
# /usr/lib/python2.6/linecache.pyc matches /usr/lib/python2.6/linecache.py
import linecache # precompiled from /usr/lib/python2.6/linecache.pyc
# /usr/lib/python2.6/types.pyc matches /usr/lib/python2.6/types.py
import types # precompiled from /usr/lib/python2.6/types.pyc
# /usr/lib/python2.6/UserDict.pyc matches /usr/lib/python2.6/UserDict.py
import UserDict # precompiled from /usr/lib/python2.6/UserDict.pyc
# /usr/lib/python2.6/_abcoll.pyc matches /usr/lib/python2.6/_abcoll.py
import _abcoll # precompiled from /usr/lib/python2.6/_abcoll.pyc
# /usr/lib/python2.6/abc.pyc matches /usr/lib/python2.6/abc.py
import abc # precompiled from /usr/lib/python2.6/abc.pyc
# /usr/lib/python2.6/copy_reg.pyc matches /usr/lib/python2.6/copy_reg.py
import copy_reg # precompiled from /usr/lib/python2.6/copy_reg.pyc
# /usr/lib/python2.6/site-packages/abrt_exception_handler.pyc matches /usr/lib/python2.6/site-packages/abrt_exception_handler.py
import abrt_exception_handler # precompiled from /usr/lib/python2.6/site-packages/abrt_exception_handler.pyc
import encodings # directory /usr/lib/python2.6/encodings
# /usr/lib/python2.6/encodings/__init__.pyc matches /usr/lib/python2.6/encodings/__init__.py
import encodings # precompiled from /usr/lib/python2.6/encodings/__init__.pyc
# /usr/lib/python2.6/codecs.pyc matches /usr/lib/python2.6/codecs.py
import codecs # precompiled from /usr/lib/python2.6/codecs.pyc
import _codecs # builtin
# /usr/lib/python2.6/encodings/aliases.pyc matches /usr/lib/python2.6/encodings/aliases.py
import encodings.aliases # precompiled from /usr/lib/python2.6/encodings/aliases.pyc
# /usr/lib/python2.6/encodings/gb18030.pyc matches /usr/lib/python2.6/encodings/gb18030.py
import encodings.gb18030 # precompiled from /usr/lib/python2.6/encodings/gb18030.pyc
dlopen("/usr/lib/python2.6/lib-dynload/_codecs_cn.so", 2);
import _codecs_cn # dynamically loaded from /usr/lib/python2.6/lib-dynload/_codecs_cn.so
dlopen("/usr/lib/python2.6/lib-dynload/_multibytecodecmodule.so", 2);
import _multibytecodec # dynamically loaded from /usr/lib/python2.6/lib-dynload/_multibytecodecmodule.so
Python 2.6.6 (r266:84292, Jun 18 2012, 14:10:23)
[GCC 4.4.6 20110731 (Red Hat 4.4.6-3)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
dlopen("/usr/lib/python2.6/lib-dynload/readline.so", 2);
import readline # dynamically loaded from /usr/lib/python2.6/lib-dynload/readline.so
>>>
（二）crtl+D，退出python命令模式。
# clear __builtin__._
# clear sys.path
# clear sys.argv
# clear sys.ps1
# clear sys.ps2
# clear sys.exitfunc
# clear sys.exc_type
# clear sys.exc_value
# clear sys.exc_traceback
# clear sys.last_type
# clear sys.last_value
# clear sys.last_traceback
# clear sys.path_hooks
# clear sys.path_importer_cache
# clear sys.meta_path
# clear sys.flags
# clear sys.float_info
# restore sys.stdin
# restore sys.stdout
# restore sys.stderr
# cleanup __main__
# cleanup[1] encodings
# cleanup[1] site
# cleanup[1] abc
# cleanup[1] _codecs
# cleanup[1] _warnings
# cleanup[1] zipimport
# cleanup[1] readline
# cleanup[1] encodings.gb18030
# cleanup[1] signal
# cleanup[1] posix
# cleanup[1] encodings.aliases
# cleanup[1] exceptions
# cleanup[1] abrt_exception_handler
# cleanup[1] _multibytecodec
# cleanup[1] codecs
# cleanup[1] _codecs_cn
# cleanup[2] copy_reg
# cleanup[2] posixpath
# cleanup[2] errno
# cleanup[2] _abcoll
# cleanup[2] types
# cleanup[2] genericpath
# cleanup[2] stat
# cleanup[2] warnings
# cleanup[2] UserDict
# cleanup[2] os.path
# cleanup[2] linecache
# cleanup[2] os
# cleanup sys
# cleanup __builtin__
# cleanup ints: 19 unfreed ints
# cleanup floats
二、spark的安装很简单，在单机单节点下，解压压缩文件，基本部署完毕。但它的运行模式很多：本地模式运行，伪分布模式运行，分布式集群的方式
等传统的，还有很多新的模式：hadoop YARN模式。。。
三、spark安装包下载的示例代码都在（/examples/src/main）目录下，可以直接调用，运行sparkip程序如下（计算一个pi值），并控制打印结果。
(一)示例代码目录
[root@localhost bin]# ls
beeline pyspark.cmd spark-class.cmd spark-shell2.cmd
beeline.cmd run-example Sparkpilog.txt spark-shell.cmd
load-spark-env.cmd run-example2.cmd sparkR spark-sql
load-spark-env.sh run-example.cmd sparkR2.cmd spark-submit
pyspark spark-class sparkR.cmd spark-submit2.cmd
pyspark2.cmd spark-class2.cmd spark-shell spark-submit.cmd
[root@localhost bin]# cd ..
[root@localhost spark-1.6.1-bin-hadoop1]# ls
bin data lib logs R sbin
CHANGES.txt ec2 LICENSE NOTICE README.md work
conf examples licenses python RELEASE
[root@localhost spark-1.6.1-bin-hadoop1]# cd examples
[root@localhost examples]# ls
src
[root@localhost examples]# cd src
[root@localhost src]# ls
main
[root@localhost src]# cd main
[root@localhost main]# ls
java python r resources scala
[root@localhost main]# cd java
[root@localhost java]# ls
org
[root@localhost java]# cd org
[root@localhost org]# ls
apache
[root@localhost org]# cd apache
[root@localhost apache]# ls
spark
[root@localhost apache]# cd saprk
bash: cd: saprk: No such file or directory
[root@localhost apache]# cd spark
[root@localhost spark]# ls
examples
[root@localhost spark]# ce examples
bash: ce: command not found
[root@localhost spark]# cd examples
[root@localhost examples]# ls
JavaHdfsLR.java JavaSparkPi.java JavaWordCount.java sql
JavaLogQuery.java JavaStatusTrackerDemo.java ml streaming
JavaPageRank.java JavaTC.java mllib
（二）运行sparkip程序如下（计算一个pi值）并控制打印结果。run-example SparkPi 10 > Sparkpilog.txt
[root@localhost spark-1.6.1-bin-hadoop1]# cd bin
[root@localhost bin]# lsbeeline pyspark2.cmd spark-class.cmd spark-sql
beeline.cmd pyspark.cmd sparkR spark-submit
derby.log run-example sparkR2.cmd spark-submit2.cmd
load-spark-env.cmd run-example2.cmd sparkR.cmd spark-submit.cmd
load-spark-env.sh run-example.cmd spark-shell
metastore_db spark-class spark-shell2.cmd
pyspark spark-class2.cmd spark-shell.cmd
[root@localhost bin]# run-example
Usage: ./bin/run-example
- set MASTER=XX to use a specific master
- can use abbreviated example class name relative to com.apache.spark.examples
(e.g. SparkPi, mllib.LinearRegression, streaming.KinesisWordCountASL)
[root@localhost bin]# ls
beeline pyspark2.cmd spark-class.cmd spark-sql
beeline.cmd pyspark.cmd sparkR spark-submit
derby.log run-example sparkR2.cmd spark-submit2.cmd
load-spark-env.cmd run-example2.cmd sparkR.cmd spark-submit.cmd
load-spark-env.sh run-example.cmd spark-shell
metastore_db spark-class spark-shell2.cmd
pyspark spark-class2.cmd spark-shell.cmd
结果：
Pi is roughly 3.142888
四、本地模式运行wordcont
[root@localhost bin]# run-example JavaWordCount input/0.txt > output.txt
16/04/30 20:18:47 INFO spark.SparkContext: Running Spark version 1.6.1
16/04/30 20:18:49 WARN util.Utils: Your hostname, localhost.localdomain resolves to a loopback address: 127.0.0.1; using 192.168.86.139 instead (on interface eth0)
16/04/30 20:18:49 WARN util.Utils: Set SPARK_LOCAL_IP if you need to bind to another address
16/04/30 20:18:49 INFO spark.SecurityManager: Changing view acls to: root
16/04/30 20:18:49 INFO spark.SecurityManager: Changing modify acls to: root
16/04/30 20:18:49 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root)
16/04/30 20:18:52 INFO util.Utils: Successfully started service 'sparkDriver' on port 37197.
16/04/30 20:18:54 INFO slf4j.Slf4jLogger: Slf4jLogger started
16/04/30 20:18:54 INFO Remoting: Starting remoting
16/04/30 20:18:55 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://[email protected]:52398]
16/04/30 20:18:55 INFO util.Utils: Successfully started service 'sparkDriverActorSystem' on port 52398.
16/04/30 20:18:55 INFO spark.SparkEnv: Registering MapOutputTracker
16/04/30 20:18:55 INFO spark.SparkEnv: Registering BlockManagerMaster
16/04/30 20:18:56 INFO storage.DiskBlockManager: Created local directory at /tmp/blockmgr-1538b230-e118-4cb8-ae03-69220d4f59e6
16/04/30 20:18:56 INFO storage.MemoryStore: MemoryStore started with capacity 517.4 MB
16/04/30 20:18:56 INFO spark.SparkEnv: Registering OutputCommitCoordinator
16/04/30 20:19:07 INFO server.Server: jetty-8.y.z-SNAPSHOT
16/04/30 20:19:08 INFO server.AbstractConnector: Started [email protected]:4040
16/04/30 20:19:08 INFO util.Utils: Successfully started service 'SparkUI' on port 4040.
16/04/30 20:19:08 INFO ui.SparkUI: Started SparkUI at http://192.168.86.139:4040
16/04/30 20:19:08 INFO spark.HttpFileServer: HTTP File server directory is /tmp/spark-debe3a1f-bbdf-40c3-981d-da572fea6a03/httpd-80fad9c1-c148-49e2-861c-e6fd1b17de5d
16/04/30 20:19:08 INFO spark.HttpServer: Starting HTTP Server
16/04/30 20:19:08 INFO server.Server: jetty-8.y.z-SNAPSHOT
16/04/30 20:19:08 INFO server.AbstractConnector: Started [email protected]:52998
16/04/30 20:19:08 INFO util.Utils: Successfully started service 'HTTP file server' on port 52998.
16/04/30 20:19:12 INFO spark.SparkContext: Added JAR file:/usr/lib/spark/spark-1.6.1-bin-hadoop1/lib/spark-examples-1.6.1-hadoop1.2.1.jar at http://192.168.86.139:52998/jars/spark-examples-1.6.1-hadoop1.2.1.jar with timestamp 1462072752175
16/04/30 20:19:12 INFO executor.Executor: Starting executor ID driver on host localhost
16/04/30 20:19:12 INFO util.Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 34480.
16/04/30 20:19:12 INFO netty.NettyBlockTransferService: Server created on 34480
16/04/30 20:19:12 INFO storage.BlockManagerMaster: Trying to register BlockManager
16/04/30 20:19:12 INFO storage.BlockManagerMasterEndpoint: Registering block manager localhost:34480 with 517.4 MB RAM, BlockManagerId(driver, localhost, 34480)
16/04/30 20:19:12 INFO storage.BlockManagerMaster: Registered BlockManager
16/04/30 20:19:17 WARN util.SizeEstimator: Failed to check whether UseCompressedOops is set; assuming yes
16/04/30 20:19:17 INFO storage.MemoryStore: Block broadcast_0 stored as values in memory (estimated size 41.9 KB, free 41.9 KB)
16/04/30 20:19:17 INFO storage.MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 4.2 KB, free 46.1 KB)
16/04/30 20:19:17 INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:34480 (size: 4.2 KB, free: 517.4 MB)
16/04/30 20:19:17 INFO spark.SparkContext: Created broadcast 0 from textFile at JavaWordCount.java:45
16/04/30 20:19:29 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/04/30 20:19:29 WARN snappy.LoadSnappy: Snappy native library not loaded
16/04/30 20:19:29 INFO mapred.FileInputFormat: Total input paths to process : 1
16/04/30 20:19:38 INFO spark.SparkContext: Starting job: collect at JavaWordCount.java:68
16/04/30 20:19:39 INFO scheduler.DAGScheduler: Registering RDD 3 (mapToPair at JavaWordCount.java:54)
16/04/30 20:19:39 INFO scheduler.DAGScheduler: Got job 0 (collect at JavaWordCount.java:68) with 1 output partitions
16/04/30 20:19:39 INFO scheduler.DAGScheduler: Final stage: ResultStage 1 (collect at JavaWordCount.java:68)
16/04/30 20:19:39 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 0)
16/04/30 20:19:39 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 0)
16/04/30 20:19:39 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[3] at mapToPair at JavaWordCount.java:54), which has no missing parents
16/04/30 20:19:39 INFO storage.MemoryStore: Block broadcast_1 stored as values in memory (estimated size 4.8 KB, free 50.9 KB)
16/04/30 20:19:39 INFO storage.MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.6 KB, free 53.5 KB)
16/04/30 20:19:39 INFO storage.BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:34480 (size: 2.6 KB, free: 517.4 MB)
16/04/30 20:19:39 INFO spark.SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1006
16/04/30 20:19:39 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[3] at mapToPair at JavaWordCount.java:54)
16/04/30 20:19:39 INFO scheduler.TaskSchedulerImpl: Adding task set 0.0 with 1 tasks
16/04/30 20:19:40 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, partition 0,PROCESS_LOCAL, 2219 bytes)
16/04/30 20:19:40 INFO executor.Executor: Running task 0.0 in stage 0.0 (TID 0)
16/04/30 20:19:40 INFO executor.Executor: Fetching http://192.168.86.139:52998/jars/spark-examples-1.6.1-hadoop1.2.1.jar with timestamp 1462072752175
16/04/30 20:19:40 INFO util.Utils: Fetching http://192.168.86.139:52998/jars/spark-examples-1.6.1-hadoop1.2.1.jar to /tmp/spark-debe3a1f-bbdf-40c3-981d-da572fea6a03/userFiles-0a6c7248-3bcd-4b94-b5ed-cb3f1394e1e9/fetchFileTemp4496955273132858252.tmp
16/04/30 20:19:47 INFO executor.Executor: Adding file:/tmp/spark-debe3a1f-bbdf-40c3-981d-da572fea6a03/userFiles-0a6c7248-3bcd-4b94-b5ed-cb3f1394e1e9/spark-examples-1.6.1-hadoop1.2.1.jar to class loader
16/04/30 20:19:47 INFO rdd.HadoopRDD: Input split: hdfs://192.168.86.139:9000/user/root/input/0.txt:0+0
16/04/30 20:19:48 INFO executor.Executor: Finished task 0.0 in stage 0.0 (TID 0). 2253 bytes result sent to driver
16/04/30 20:19:49 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 9059 ms on localhost (1/1)
16/04/30 20:19:49 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool
16/04/30 20:19:49 INFO scheduler.DAGScheduler: ShuffleMapStage 0 (mapToPair at JavaWordCount.java:54) finished in 9.298 s
16/04/30 20:19:49 INFO scheduler.DAGScheduler: looking for newly runnable stages
16/04/30 20:19:49 INFO scheduler.DAGScheduler: running: Set()
16/04/30 20:19:49 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 1)
16/04/30 20:19:49 INFO scheduler.DAGScheduler: failed: Set()
16/04/30 20:19:49 INFO scheduler.DAGScheduler: Submitting ResultStage 1 (ShuffledRDD[4] at reduceByKey at JavaWordCount.java:61), which has no missing parents
16/04/30 20:19:49 INFO storage.MemoryStore: Block broadcast_2 stored as values in memory (estimated size 2.9 KB, free 56.4 KB)
16/04/30 20:19:49 INFO storage.MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 1752.0 B, free 58.2 KB)
16/04/30 20:19:49 INFO storage.BlockManagerInfo: Added broadcast_2_piece0 in memory on localhost:34480 (size: 1752.0 B, free: 517.4 MB)
16/04/30 20:19:49 INFO spark.SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1006
16/04/30 20:19:49 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 1 (ShuffledRDD[4] at reduceByKey at JavaWordCount.java:61)
16/04/30 20:19:49 INFO scheduler.TaskSchedulerImpl: Adding task set 1.0 with 1 tasks
16/04/30 20:19:49 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, localhost, partition 0,PROCESS_LOCAL, 1973 bytes)
16/04/30 20:19:49 INFO executor.Executor: Running task 0.0 in stage 1.0 (TID 1)
16/04/30 20:19:49 INFO storage.ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 1 blocks
16/04/30 20:19:49 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 89 ms
16/04/30 20:19:49 INFO executor.Executor: Finished task 0.0 in stage 1.0 (TID 1). 1161 bytes result sent to driver
16/04/30 20:19:50 INFO scheduler.DAGScheduler: ResultStage 1 (collect at JavaWordCount.java:68) finished in 0.554 s
16/04/30 20:19:50 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 555 ms on localhost (1/1)
16/04/30 20:19:50 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool
16/04/30 20:19:50 INFO scheduler.DAGScheduler: Job 0 finished: collect at JavaWordCount.java:68, took 11.299262 s
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/metrics/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/api,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/static,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/json,null}
16/04/30 20:19:50 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs,null}
16/04/30 20:19:50 INFO ui.SparkUI: Stopped Spark web UI at http://192.168.86.139:4040
16/04/30 20:19:50 INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
16/04/30 20:19:50 INFO storage.MemoryStore: MemoryStore cleared
16/04/30 20:19:50 INFO storage.BlockManager: BlockManager stopped
16/04/30 20:19:50 INFO storage.BlockManagerMaster: BlockManagerMaster stopped
16/04/30 20:19:50 INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
16/04/30 20:19:50 INFO spark.SparkContext: Successfully stopped SparkContext
16/04/30 20:19:50 INFO remote.RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.
16/04/30 20:19:50 INFO util.ShutdownHookManager: Shutdown hook called
16/04/30 20:19:51 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-debe3a1f-bbdf-40c3-981d-da572fea6a03
16/04/30 20:19:51 INFO remote.RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.
16/04/30 20:19:51 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-debe3a1f-bbdf-40c3-981d-da572fea6a03/httpd-80fad9c1-c148-49e2-861c-e6fd1b17de5d
[root@localhost spark-1.6.1-bin-hadoop1]# cd bin
[root@localhost bin]# ls
beeline pyspark.cmd Sparkpilog.txt spark-sql
beeline.cmd run-example sparkR spark-submit
load-spark-env.cmd run-example2.cmd sparkR2.cmd spark-submit2.cmd
load-spark-env.sh run-example.cmd sparkR.cmd spark-submit.cmd
output.txt spark-class spark-shell
pyspark spark-class2.cmd spark-shell2.cmd
pyspark2.cmd spark-class.cmd spark-shell.cmd
[root@localhost bin]# gedit output.txt
started;: 1
scala: 1
created: 1
pyspark.cmd: 2
load-spark-env.cmd: 2
spark: 2
sparkR2.cmd: 2
listening: 1
'sparkDriver': 1
[root@localhost: 24
MapOutputTracker: 1
Remoting: 1
spark-submit2.cmd: 2
o.s.j.s.ServletContextHandler{/executors,null}: 1
o.s.j.s.ServletContextHandler{/stages/stage,null}: 1
WARN: 5
broadcast_0: 1
ce:: 1
ID: 1
instead: 1
47539.: 1
o.s.j.s.ServletContextHandler{/static,null}: 1
version: 1
file: 2
Unable: 1
stop(): 1
address:: 1
org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206): 1
cd:: 1
host: 1
shut: 1
timestamp: 1
server.Server:: 2
46.1: 1
JavaWordCount: 1
org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:197): 1
values: 1
http://192.168.86.139:43838/jars/spark-examples-1.6.1-hadoop1.2.1.jar: 1
ec2: 1
org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237): 4
java: 2
Added: 2
JAR: 1
ml: 1
localhost: 1
/tmp/blockmgr-77d5002a-93cb-4d9c-849d-51a41b7c927f: 1
[email protected]:4040: 1
proceeding: 1
size: 2
Sparkpilog.txt: 3
spark]#: 3
o.s.j.s.ServletContextHandler{/storage,null}: 1
localhost:34253: 2
found: 1
KB): 2
Successfully: 6
CHANGES.txt: 1
Deleting: 2
517.4: 3
19:58:55: 3
Registering: 4
spark-class.cmd: 2
examples: 5
at: 5
o.s.j.s.ServletContextHandler{/jobs/job,null}: 1
util.Utils:: 7
localhost,: 1
Block: 2
JavaWordCount.java:45: 1
o.s.j.s.ServletContextHandler{/api,null}: 1
'sparkDriverActorSystem': 1
UI: 1
mllib: 1
o.s.j.s.ServletContextHandler{/storage/rdd,null}: 1
org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239): 4
spark-sql: 2
stopped!: 2
disabled;: 2
(estimated: 2
command: 1
main: 2
cd: 11
spark-shell: 2
or: 1
org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181): 1
need: 1
storage.BlockManager:: 1
Remoting:: 2
http://192.168.86.139:4040: 2
eth0): 1
o.s.j.s.ServletContextHandler{/jobs/job/json,null}: 1
does: 1
JavaStatusTrackerDemo.java: 1
'org.apache.spark.network.netty.NettyBlockTransferService': 1
spark-submit.cmd: 2
RELEASE: 1
storage.BlockManagerMaster:: 3
native: 1
at: 34
remote: 2
remoting: 1
127.0.0.1;: 1
JavaPageRank.java: 1
util.ShutdownHookManager:: 3
Stopped: 1
INFO: 72
No: 1
org: 2
main]#: 2
for: 1
o.s.j.s.ServletContextHandler{/stages/pool/json,null}: 1
Slf4jLogger: 1
capacity: 1
resources: 1
OutputCommitCoordinator: 2
/tmp/spark-a9679d37-7536-45fd-87de-12917189c3e7: 1
localhost.localdomain: 1
BlockManagerId(driver,: 1
loaded: 1
[email protected]:43838: 1
run-example: 3
broadcast: 1
resolves: 1
executor.Executor:: 1
19:59:10: 2
34253): 1
57283.: 1
sun.reflect.NativeMethodAccessorImpl.invoke0(Native: 1
started: 7
MB): 1
file:/usr/lib/spark/spark-1.6.1-bin-hadoop1/lib/spark-examples-1.6.1-hadoop1.2.1.jar: 1
..: 1
Method): 1
builtin-java: 1
r: 1
free:: 1
exist:: 1
broadcast_0_piece0: 2
org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35): 3
Snappy: 1
ls: 11
Invoking: 1
Registered: 1
bind: 1
spark.SecurityManager:: 3
spark-shell.cmd: 2
modify: 2
scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint:: 1
classes: 1
RAM,: 1
storage.DiskBlockManager:: 1
1462071550387: 1
service: 5
in: 4
'HTTP: 1
19:58:52: 1
JavaLogQuery.java: 1
Failed: 1
server': 1
shutdown: 1
from: 2
Running: 1
spark.SparkContext:: 5
src]#: 2
Exception: 1
Set: 1
34253: 1
UseCompressedOops: 1
interface: 1
textFile: 1
spark.HttpFileServer:: 1
./wordcountdata.txt: 1
o.s.j.s.ServletContextHandler{/jobs/json,null}: 1
storage.BlockManagerInfo:: 1
pyspark: 2
JavaHdfsLR.java: 1
"main": 1
bin: 2
Input: 1
org.apache.spark.api.java.JavaPairRDD.reduceByKey(JavaPairRDD.scala:526): 1
org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:199): 1
util.SizeEstimator:: 1
Remote: 1
applicable: 1
beeline: 2
: 304
apache]#: 3
SPARK_LOCAL_IP: 1
users: 2
19:59:11: 5
flushing: 1
o.s.j.s.ServletContextHandler{/,null}: 1
16/04/30: 77
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43): 1
org.apache.spark.Partitioner$.defaultPartitioner(Partitioner.scala:65): 1
19:58:49: 1
netty.NettyBlockTransferService:: 1
o.s.j.s.ServletContextHandler{/metrics/json,null}: 1
o.s.j.s.ServletContextHandler{/stages/pool,null}: 1
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala): 1
conf: 1
register: 1
such: 1
4040.: 1
spark-class: 2
JavaSparkPi.java: 1
sparkR: 2
scala.Option.getOrElse(Option.scala:120): 4
hook: 2
Trying: 1
org]#: 2
ui.SparkUI:: 2
Changing: 2
set;: 1
o.s.j.s.ServletContextHandler{/executors/json,null}: 1
remote.RemoteActorRefProvider$RemotingTerminator:: 2
called: 1
sbin: 1
acls: 3
free: 2
licenses: 1
not: 3
java]#: 2
address: 1
if: 1
transports.: 1
ce: 1
thread: 1
as: 2
o.s.j.s.ServletContextHandler{/stages/json,null}: 1
File: 1
slf4j.Slf4jLogger:: 1
storage.MemoryStore:: 4
41.9: 2
o.s.j.s.ServletContextHandler{/storage/json,null}: 1
stored: 2
o.s.j.s.ServletContextHandler{/jobs,null}: 1
authentication: 1
is: 2
R: 1
spark-class2.cmd: 2
sparkR.cmd: 2
on: 9
check: 1
JavaTC.java: 1
4.2: 2
down;: 1
spark-1.6.1-bin-hadoop1]#: 3
spark.MapOutputTrackerMasterEndpoint:: 1
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731): 1
using: 2
beeline.cmd: 2
LICENSE: 1
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57): 1
/tmp/spark-a9679d37-7536-45fd-87de-12917189c3e7/httpd-11d327ec-8c66-4c85-bcd2-698a68f3a5c4: 2
root: 2
src: 2
native-hadoop: 1
BlockManager: 3
o.s.j.s.ServletContextHandler{/environment,null}: 1
saprk:: 1
HTTP: 2
to:: 2
ui: 1
directory: 5
executor: 1
NOTICE: 1
Started: 3
lib: 1
bin]#: 4
logs: 1
MapOutputTrackerMasterEndpoint: 1
apache: 2
(on: 1
192.168.86.139: 1
server.AbstractConnector:: 2
spark-shell2.cmd: 2
'SparkUI': 1
Starting: 3
:[akka.tcp://[email protected]:47539]: 1
o.s.j.s.ServletContextHandler{/stages/stage/json,null}: 1
Spark: 2
jetty-8.y.z-SNAPSHOT: 2
platform...: 1
hostname,: 1
permissions:: 2
addresses: 1
library: 2
saprk: 1
19:59:14: 5
README.md: 1
1.6.1: 1
19:59:07: 9
loopback: 1
whether: 1
19:58:56: 4
org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:208): 1
bash:: 2
Server: 2
o.s.j.s.ServletContextHandler{/executors/threadDump/json,null}: 1
cleared: 1
sql: 1
your: 1
SparkContext: 1
load-spark-env.sh: 2
org.apache.hadoop.mapred.InvalidInputException:: 1
load: 1
Shutdown: 1
run-example.cmd: 2
spark.HttpServer:: 1
>: 1
Shutting: 1
19:58:54: 2
assuming: 1
path: 1
JavaWordCount.java: 1
o.s.j.s.ServletContextHandler{/executors/threadDump,null}: 1
streaming: 1
snappy.LoadSnappy:: 1
43838.: 1
Created: 2
driver: 1
BlockManagerMaster: 2
o.s.j.s.ServletContextHandler{/stages,null}: 1
util.NativeCodeLoader:: 1
o.s.j.s.ServletContextHandler{/storage/rdd/json,null}: 1
stopped: 28
storage.BlockManagerMasterEndpoint:: 1
19:58:50: 5
handler.ContextHandler:: 25
with: 6
data: 1
java.lang.reflect.Method.invoke(Method.java:606): 1
org.apache.spark.rdd.RDD.partitions(RDD.scala:237): 4
SparkUI: 1
Set(root): 1
19:59:26: 34
block: 1
hdfs://192.168.86.139:9000/user/root/Sparkpilog.txt: 1
19:59:27: 6
bytes: 1
port: 5
Set(root);: 1
view: 2
(size:: 1
daemon.: 1
server: 1
you: 1
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121): 1
yes: 1
another: 1
spark-submit: 2
a: 1
memory: 3
SecurityManager:: 1
down: 1
0: 1
work: 1
local: 1
manager: 1
python: 2
to: 6
spark.SparkEnv:: 3
org.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:61): 1
web: 1
MB: 2
run-example2.cmd: 2
Your: 1
where: 1
o.s.j.s.ServletContextHandler{/environment/json,null}: 1
daemon: 1
pyspark2.cmd: 2
KB,: 3
MemoryStore: 2
o.s.j.s.ServletContextHandler{/stages/stage/kill,null}: 1
examples]#: 3
34253.: 1