Spark 单节点配置

1. Spark configuration

1.1 Configure Environment

1.1.1 Edit .bashrc
vi ~/.bashrc

#JAVA ENV
export JAVA_HOME=~/Java
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:JRE_HOME/lib
export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin

#Hadoop ENV
export HADOOP_HOME=~/Hadoop
export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native

#Sprak ENV
export SPARK_HOME=~/Spark/
export PATH=$PATH:$SPARK_HOME/bin

source ~/.bashrc

1.1.2 Edit spark-env.sh

cd ~/Spark/conf/
cp spark-env.sh.template spark-env.sh
vi spark-env.sh

export SPARK_WORKER_CORES=2
export SPARK_WORKER_MEMORY=4g
export SPARK_WORKER_INSTANCES=2
export SPARK_EXECUTOR_MEMORY=2g
export SPARK_DRIVE_MEMORY=2g

---

1.2 Edit host file

sudo vi /etc/hosts

127.0.0.1 namenode

---

1.3 Edit core-site.xml

cd ~/Hadoop/etc/hadoop/
vi core-site.xml

    fs.defaultFS
    hdfs://namenode:9000

---

1.4 Edit hdfs-site.xml

vi hdfs-site.xml

    dfs.replication
    1


    dfs.name.dir
    /home/huang/data/namenode


    dfs.data.dir
    /home/huang/data/datanode


    dfs.namenode.secondary.http-address
    namenode:50090


    dfs.webhdfs.enabled
    true

---

1.5 Edit mapred-site.xml

cp mapred-site.xml.template mapred-site.xml
vi mapred-site.xml

    fs.defaultFS
    hdfs://namenode:9000

---

1.6 Edit yarn-site.xml

vi yarn-site.xml

    yarn.nodemanager.aux-services
    mapreduce_shuffle


    yarn.nodemanager.resource.memory-mb
    9124


    yarn.nodemanager.resource.cpu-vcores
    4

---

1.7 Edit slave

vi slaves

localhost