Spark集群安装笔记

# HADOOP VARIABLES START
export JAVA_HOME=/usr/java
export JRE_HOME=$JAVA_HOME/jre
export SCALA_HOME=/usr/scala-2.11.6
export MAVEN_HOME=/usr/apache-maven-3.3.9
export ANT_HOME=/usr/apache-ant-1.9.6
export SPARK_HOME=/usr/hadoop/spark-1.6.1-bin-hadoop2.6
export SPARK_JAR=/usr/hadoop/spark-1.6.1-bin-hadoop2.6/lib/spark-assembly-1.6.0-cdh5.7.1-hadoop2.6.0-cdh5.7.1.jar
export CLASSPATH=$JAVA_HOME/lib:$JAVA_HOME/jre/lib
export HADOOP_HOME=/usr/hadoop/hadoop-2.6.4
export PATH=$PATH:$HADOOP_HOME/bin:$JAVA_HOME/bin:$HADOOP_HOME/sbin:$SCALA_HOME/bin:$SPARK_HOME/bin:$SPARK_HOME/sbin:${MAVEN_HOME}/bin:/softback/sbt/bin:$ANT_HOME/bin
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
# HADOOP VARIABLES END


------------ssh免登陆------
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
cat ~/.ssh/id_dsa.pub | ssh [email protected] 'cat >> ~/.ssh/authorized_keys'
cat ~/.ssh/id_dsa.pub | ssh [email protected] 'cat >> ~/.ssh/authorized_keys'


cat ~/.ssh/id_dsa.pub | ssh [email protected] 'cat >> ~/.ssh/authorized_keys'
cat ~/.ssh/id_dsa.pub | ssh [email protected] 'cat >> ~/.ssh/authorized_keys'


cat ~/.ssh/id_dsa.pub | ssh [email protected] 'cat >> ~/.ssh/authorized_keys'
cat ~/.ssh/id_dsa.pub | ssh [email protected] 'cat >> ~/.ssh/authorized_keys'
----------------------------------------------
scp -r /usr/scala-2.11.6 [email protected]:/usr/scala-2.11.6
scp -r /usr/scala-2.11.6 [email protected]:/usr/scala-2.11.6


scp -r /etc/profile [email protected]:/etc/profile
scp -r /etc/profile [email protected]:/etc/profile




scp -r /usr/hadoop/hadoop-2.6.4 [email protected]:/usr/hadoop/hadoop-2.6.4
scp -r /usr/hadoop/hadoop-2.6.4 [email protected]:/usr/hadoop/hadoop-2.6.4


scp -r /usr/hadoop/spark-1.6.1-bin-hadoop2.6 [email protected]:/usr/hadoop/spark-1.6.1-bin-hadoop2.6
scp -r /usr/hadoop/spark-1.6.1-bin-hadoop2.6 [email protected]:/usr/hadoop/spark-1.6.1-bin-hadoop2.6


scp -r /usr/java [email protected]:/usr/java
scp -r /usr/java [email protected]:/usr/java


scp -f /etc/hosts [email protected]:/etc/hosts
scp -f /etc/hosts [email protected]:/etc/hosts


scp -r /usr/hadoop/spark-1.6.1-bin-hadoop2.6/conf/spark-env.sh [email protected]:/usr/hadoop/spark-1.6.1-bin-hadoop2.6/conf/spark-env.sh
scp -r /usr/hadoop/spark-1.6.1-bin-hadoop2.6/conf/spark-env.sh [email protected]:/usr/hadoop/spark-1.6.1-bin-hadoop2.6/conf/spark-env.sh


--------------------------------------------------------------
mkdir -p /data/hadoop/namenode
mkdir -p /data/hadoop/datanode
mkdir -p /data/hadoop/hadoop_tmp


rm -rf /data/hadoop/namenode/*
rm -rf /data/hadoop/datanode/*
rm -rf /data/hadoop/hadoop_tmp/*


------------------------------
-----core-site.xml--------------------------



        fs.defaultFS
        hdfs://master:9000


        io.file.buffer.size
        131072


        hadoop.tmp.dir
        /data/hadoop/hadoop_tmp
        Abasefor other temporary directories.


        hadoop.proxyuser.spark.hosts
        *


        hadoop.proxyuser.spark.groups
        *




-------hdfs-site.xml------


        dfs.namenode.secondary.http-address
        master:9001


        dfs.namenode.name.dir
        /data/hadoop/namenode


        dfs.datanode.data.dir
        /data/hadoop/datanode


        dfs.replication
        1


        dfs.webhdfs.enabled
        true


        dfs.datanode.du.reserved
        10240000000






-----------yarn-site.xml--------------------------------


   
        yarn.nodemanager.aux-services
        mapreduce_shuffle
   
   
        yarn.nodemanager.aux-services.mapreduce.shuffle.class
        org.apache.hadoop.mapred.ShuffleHandler
   
   
        yarn.resourcemanager.address
        master:8032
   
   
        yarn.resourcemanager.scheduler.address
        master:8030
   
   
        yarn.resourcemanager.resource-tracker.address
        master:8035
   
   
        yarn.resourcemanager.admin.address
        master:8033
   
   
        yarn.resourcemanager.webapp.address
        master:8088
   

----------mapred-site.xml----------------------------------------------------


        mapreduce.framework.name
        yarn


        mapreduce.jobhistory.address
        master:10020


        mapreduce.jobhistory.webapp.address
        master:19888


---------------------------------------------------------------------
hadoop namenode -format 
---------启动HDFS------------------
start-dfs.sh
-------------------------------------------


-----------启动YARN-----------
start-yarn.sh
--------------spark spark-env.sh文件 在最下面增加------------------------------
----------
JAVA_HOME=/usr/java
SCALA_HOME=/usr/scala-2.11.6
SPARK_MASTER_IP=master
SPARK_PORT=7077
SPARK_CORES=1
SPARK_WORKER_MEMORY=1g
SPARK_WORKER_PORT=7078
SPARK_WORKER_WEBUI_PORT=8081
SPARK_WORKER_INSTANCES=1
HADOOP_CONF_DIR=/usr/hadoop/hadoop-2.6.4/etc/hadoop


SPARK_HISTORY_OPTS="-Dspark.history.fs.logDirectory=hdfs://master:9000/spark/history"
------spark-defaults.conf----------------------------------------------------------------------------
spark.master                     spark://master:7077
spark.eventLog.enabled           true
spark.eventLog.dir               hdfs://master:9000/spark/history
spark.eventLog.compress           true
------------------------------------------------------------------------------------------------------
export MAVEN_OPTS="-Xms256m -Xmx512m"
mvn clean package -DskipTests -Pdist,native -Dtar


yum -y install gcc*
yum -y install cmake
yum -y install glibc-headers
yum -y install gcc-c++
yum -y install zip-devel
yum -y install openssl-devel
yum -y install svn
yum -y install git
yum -y install ncurses-devel 
yum -y install lzo-devel
yum -y install autoconf
yum -y install libtool
yum -y install automake
----------配置jdk 指向-----------------
which java
alternatives --install /usr/bin/java java /usr/java/bin/java 300
alternatives --config java
------------
/softback/hadoop-2.6.4-src/hadoop-dist/target/hadoop-2.6.4.tar.gz
--------------启动 spark---------------
spark-shell --executor-memory 2g --driver-memory 1g --master spark://master:7077


spark-shell --master local[2]


--------------------测试-------------------------
val rdd = sc.textFile("hdfs://192.168.1.30:9000/spark/test.txt")
rdd.cache()
val wordcount = rdd.flatMap(_.split(" ")).map(x=>(x,1)).reduceByKey(_+_)
wordcount.take(10)
val wordsort = wordcount.map(x=>(x._2,x._1)).sortByKey(false).map(x=>(x._2,x._1))
wordsort.take(10)


--------------------------
http://plugins.jetbrains.com/files/1347/25886/scala-intellij-bin-3.0.6.zip
------------web 访问地址-------------------
http://master:8080
http://master:4040
----------hdfs访问地址--------------------------------
http://master:50070
-------------spark history执行历史查询---------------------------------------------
http://master:18080
-------------------------------------------------------------------------------
-------------------idea spark 项目导入----------------------
VCS→Check out from Version Control→Git
https://github.com/apache/spark.git
---------------------------