【Spark】on Yarn 模式安装配置

在安装spark前先安装scala

下载Scala

上传到/opt/modules/softwares目录并解压到/opt/modules目录下

root@master:/opt/softwares# ll
total 745404
drwxr-xr-x 2 root root      4096 6月  12 17:08 ./
drwxr-xr-x 7 root root      4096 6月   6 16:52 ../
-rw-r--r-- 1 root root 232027212 5月  21 19:11 apache-hive-2.3.5-bin.tar.gz
-rw-r--r-- 1 root root 168663748 5月  27 22:00 apache-storm-1.2.2.tar.gz
-rw-r--r-- 1 root root 113175674 5月  20 18:50 hbase-1.4.9-bin.tar.gz
-rw-r--r-- 1 root root  19413106 6月  12 17:08 scala-2.13.0.tgz
-rw-r--r-- 1 root root 229988313 6月  12 16:58 spark-2.4.3-bin-hadoop2.7.tgz
root@master:/opt/softwares# tar -zxvf scala-2.13.0.tgz -C /opt/modules/

配置环境变量

vim ~/.bashrc

添加如下

刷新生效
source ~/.bashrc
验证

root@master:/opt/modules# scala -version
Scala code runner version 2.13.0 -- Copyright 2002-2019, LAMP/EPFL and Lightbend, Inc.

安装spark

下载Spark
上传到/opt/modules/softwares目录并解压到/opt/modules目录下

root@master:/opt/softwares# ll
total 745404
drwxr-xr-x 2 root root      4096 6月  12 17:08 ./
drwxr-xr-x 7 root root      4096 6月   6 16:52 ../
-rw-r--r-- 1 root root 232027212 5月  21 19:11 apache-hive-2.3.5-bin.tar.gz
-rw-r--r-- 1 root root 168663748 5月  27 22:00 apache-storm-1.2.2.tar.gz
-rw-r--r-- 1 root root 113175674 5月  20 18:50 hbase-1.4.9-bin.tar.gz
-rw-r--r-- 1 root root  19413106 6月  12 17:08 scala-2.13.0.tgz
-rw-r--r-- 1 root root 229988313 6月  12 16:58 spark-2.4.3-bin-hadoop2.7.tgz
root@master:/opt/softwares# tar -zxvf spark-2.4.3-bin-hadoop2.7.tgz  -C /opt/modules/

为了方便，重命名一下

root@master:/opt/modules# ll
total 36
drwxr-xr-x  9 root   root      4096 6月  12 17:11 ./
drwxr-xr-x  7 root   root      4096 6月   6 16:52 ../
drwxr-xr-x 11 root   root      4096 6月   6 13:54 hadoop-2.8.5/
drwxr-xr-x  8 root   root      4096 5月  20 19:53 hbase-1.4.9/
drwxr-xr-x  7 root   root      4096 5月  19 19:00 jdk1.8.0_201/
drwxrwxr-x  6   2000      2000 4096 6月   7 19:14 scala-2.13.0/
drwxr-xr-x 13 master ubuntu-01 4096 5月   1 13:19 spark-2.4.3/
drwxr-xr-x 15 root   root      4096 5月  27 23:16 storm-1.2.2/
drwxr-xr-x 15 root   root      4096 5月  19 20:34 zookeeper-3.4.14/
root@master:/opt/modules# mv spark-2.4.3-bin-hadoop2.7/ spark-2.4.3

编辑spark-env.sh文件，添加如下

root@master:/opt/modules/spark-2.4.3/conf# ll
total 44
drwxr-xr-x  2 master ubuntu-01 4096 5月   1 13:19 ./
drwxr-xr-x 13 master ubuntu-01 4096 5月   1 13:19 ../
-rw-r--r--  1 master ubuntu-01  996 5月   1 13:19 docker.properties.template
-rw-r--r--  1 master ubuntu-01 1105 5月   1 13:19 fairscheduler.xml.template
-rw-r--r--  1 master ubuntu-01 2025 5月   1 13:19 log4j.properties.template
-rw-r--r--  1 master ubuntu-01 7801 5月   1 13:19 metrics.properties.template
-rw-r--r--  1 master ubuntu-01  865 5月   1 13:19 slaves.template
-rw-r--r--  1 master ubuntu-01 1292 5月   1 13:19 spark-defaults.conf.template
-rwxr-xr-x  1 master ubuntu-01 4221 5月   1 13:19 spark-env.sh.template*
root@master:/opt/modules/spark-2.4.3/conf# cp spark-env.sh.template spark-env.sh
root@master:/opt/modules/spark-2.4.3/conf# ll
total 52
drwxr-xr-x  2 master ubuntu-01 4096 6月  12 17:37 ./
drwxr-xr-x 13 master ubuntu-01 4096 5月   1 13:19 ../
-rw-r--r--  1 master ubuntu-01  996 5月   1 13:19 docker.properties.template
-rw-r--r--  1 master ubuntu-01 1105 5月   1 13:19 fairscheduler.xml.template
-rw-r--r--  1 master ubuntu-01 2025 5月   1 13:19 log4j.properties.template
-rw-r--r--  1 master ubuntu-01 7801 5月   1 13:19 metrics.properties.template
-rw-r--r--  1 master ubuntu-01  865 5月   1 13:19 slaves.template
-rw-r--r--  1 master ubuntu-01 1292 5月   1 13:19 spark-defaults.conf.template
-rwxr-xr-x  1 root   root      4221 6月  12 17:37 spark-env.sh*
-rwxr-xr-x  1 master ubuntu-01 4221 5月   1 13:19 spark-env.sh.template*
root@master:/opt/modules/spark-2.4.3/conf# vim spark-env.sh

编辑yarn-site.xml文件，添加如下

 
     yarn.nodemanager.pmem-check-enabled
     false
 

 
 
     yarn.nodemanager.vmem-check-enabled
     false
 

分发到其他节点

scp yarn-site.xml root@slave01:/opt/modules/hadoop-2.8.5/etc/hadoop/
scp yarn-site.xml root@slave02:/opt/modules/hadoop-2.8.5/etc/hadoop/
scp yarn-site.xml root@slave03:/opt/modules/hadoop-2.8.5/etc/hadoop/
scp yarn-site.xml root@slave04:/opt/modules/hadoop-2.8.5/etc/hadoop/
scp yarn-site.xml root@slave05:/opt/modules/hadoop-2.8.5/etc/hadoop/
scp yarn-site.xml root@slave06:/opt/modules/hadoop-2.8.5/etc/hadoop/

启动HDFS和YARN

root@master:/opt/modules/hadoop-2.8.5/etc/hadoop# start-dfs.sh
Starting namenodes on [master]
master: starting namenode, logging to /opt/modules/hadoop-2.8.5/logs/hadoop-root-namenode-master.out
master: starting datanode, logging to /opt/modules/hadoop-2.8.5/logs/hadoop-root-datanode-master.out
slave01: starting datanode, logging to /opt/modules/hadoop-2.8.5/logs/hadoop-root-datanode-slave01.out
slave03: starting datanode, logging to /opt/modules/hadoop-2.8.5/logs/hadoop-root-datanode-slave03.out
slave05: starting datanode, logging to /opt/modules/hadoop-2.8.5/logs/hadoop-root-datanode-slave05.out
slave02: starting datanode, logging to /opt/modules/hadoop-2.8.5/logs/hadoop-root-datanode-slave02.out
slave06: starting datanode, logging to /opt/modules/hadoop-2.8.5/logs/hadoop-root-datanode-slave06.out
slave04: starting datanode, logging to /opt/modules/hadoop-2.8.5/logs/hadoop-root-datanode-slave04.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /opt/modules/hadoop-2.8.5/logs/hadoop-root-secondarynamenode-master.out
root@master:/opt/modules/hadoop-2.8.5/etc/hadoop# start-yarn.sh
starting yarn daemons
starting resourcemanager, logging to /opt/modules/hadoop-2.8.5/logs/yarn-root-resourcemanager-master.out
master: starting nodemanager, logging to /opt/modules/hadoop-2.8.5/logs/yarn-root-nodemanager-master.out
slave03: starting nodemanager, logging to /opt/modules/hadoop-2.8.5/logs/yarn-root-nodemanager-slave03.out
slave06: starting nodemanager, logging to /opt/modules/hadoop-2.8.5/logs/yarn-root-nodemanager-slave06.out
slave01: starting nodemanager, logging to /opt/modules/hadoop-2.8.5/logs/yarn-root-nodemanager-slave01.out
slave04: starting nodemanager, logging to /opt/modules/hadoop-2.8.5/logs/yarn-root-nodemanager-slave04.out
slave02: starting nodemanager, logging to /opt/modules/hadoop-2.8.5/logs/yarn-root-nodemanager-slave02.out
slave05: starting nodemanager, logging to /opt/modules/hadoop-2.8.5/logs/yarn-root-nodemanager-slave05.out

使spark-shell运行在YARN上

root@master:/opt/modules/hadoop-2.8.5/etc/hadoop# spark-shell --master yarn --deploy-mode client
19/06/12 18:19:46 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
19/06/12 18:19:50 WARN yarn.Client: Neither spark.yarn.jars nor spark.yarn.archive is set, falling back to uploading libraries under SPARK_HOME.
Spark context Web UI available at http://master:4040
Spark context available as 'sc' (master = yarn, app id = application_1560334779290_0001).
Spark session available as 'spark'.
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 2.4.3
      /_/
         
Using Scala version 2.11.12 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_201)
Type in expressions to have them evaluated.
Type :help for more information.

scala> 

查看YARN WEB
http://master:8088/cluster

单机ID查看详细信息

单击“ApplicationMaster”链接

运行程序

参考：
https://blog.csdn.net/chengyuqiang/article/details/77864246
https://blog.csdn.net/qq_28652401/article/details/83507742