虚拟机下需要
1.关闭防火墙、2.关闭selinux
systemctl status fifirewalld
vi /etc/selinux/config
3.创建目录
mkdir -p /opt/lagou/software
mkdir -p /opt/lagou/servers
4.下载jdk
cd /opt/lagou/software
wget https://download.java.net/openjdk/jdk8u41/ri/openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz
//wget https://archive.apache.org/dist/spark/spark-2.4.5/spark-2.4.5-bin-without-hadoop-scala-2.12.tgz
5.解压并移动
tar -zxvf openjdk-8u41-b04-linux-x64-14_jan_2020.tar.gz
mv java-se-8u41-ri/ ../servers/jdk1.8.0_231
6.配置环境变量
vim /etc/profile
JAVA_HOME export JAVA_HOME=/opt/lagou/servers/jdk1.8.0_231 export PATH=:JAVA_HOME/bin
source /etc/profile
7.验证jdk安装
java -version
8.安装rz sudo yum install lrzsz -y
9.上传hadoop安装包
wget https://mirrors.tuna.tsinghua.edu.cn/apache/hadoop/common/hadoop-2.9.2/hadoop-2.9.2.tar.gz
tar -zxvf hadoop-2.9.2.tar.gz -C ../servers/
vim /etc/profile
HADOOP_HOME export HADOOP_HOME=/opt/lagou/servers/hadoop-2.9.2 export PATH=:HADOOP_HOME/bin export PATH=:HADOOP_HOME/sbin
source /etc/profile
10.添加java
echo "export JAVA_HOME=/opt/lagou/servers/jdk1.8.0_231" >> /opt/lagou/servers/hadoop-2.9.2/etc/hadoop/yarn-env.sh
echo "export JAVA_HOME=/opt/lagou/servers/jdk1.8.0_231" >> /opt/lagou/servers/hadoop-2.9.2/etc/hadoop/hadoop-env.sh
11.修改主机名称
编辑/etc/cloud/cloud.cfg文件，将preserve_hostname配置项的值由false修改为true，保存并退出
在实例中修改
hostnamectl set-hostname *****
12.生成公钥
cd ~
ssh-keygen -t rsa -P ""
cat id_rsa.pub >> authorized_keys
scp authorized_keys master:/root/.ssh/tmp
cat tmp >> authorized_keys
13.mkdir -p /var/lib/hadoop-hdfs
14.配置core-site.xml
fs.defaultFS
hdfs://master:9000
hadoop.tmp.dir
/opt/lagou/servers/hadoop-2.9.2/data/tmp
hadoop.proxyuser.root.hosts
hadoop.proxyuser.root.groups
hadoop.proxyuser.hadoop.hosts
hadoop.proxyuser.hadoop.groups
15.mapred-site.xml
mapreduce.framework.name
yarn
mapreduce.jobhistory.address
master:10020
mapreduce.jobhistory.webapp.address
master:19888
16.yarn-site.xml
yarn.resourcemanager.hostname
slave1
yarn.nodemanager.aux-services
mapreduce_shuffle
yarn.log-aggregation-enable
true
yarn.log-aggregation.retain-seconds
slave1
chown -R root:root /opt/lagou/servers/hadoop-2.9.2
17.hdfs配置
18.slave文件配置
19.创建hdfs文件夹
scp core-site.xml slave1:/opt/lagou/servers/hadoop-2.9.2/etc/hadoop/
scp hdfs-site.xml slave1:/opt/lagou/servers/hadoop-2.9.2/etc/hadoop/
scp yarn-site.xml slave1:/opt/lagou/servers/hadoop-2.9.2/etc/hadoop/
scp mapred-site.xml slave1:/opt/lagou/servers/hadoop-2.9.2/etc/hadoop/
scp slaves slave1:/opt/lagou/servers/hadoop-2.9.2/etc/hadoop/
20.spark安装
cd /opt/lagou/software/
tar zxvf spark-2.4.5-bin-without-hadoop-scala-2.12.tgz
mv spark-2.4.5-bin-without-hadoop-scala-2.12/ ../servers/spark-2.4.5/
21.cd $SPARK_HOME/conf slaves、spark-defaults.conf、spark-env.sh、log4j.properties
1）修改slaves
master
slave1
slave2
slave3
2)修改spark-defaults.conf
spark.master spark://master:7077
spark.eventLog.enabled true
spark.eventLog.dir hdfs://master:9000/spark-eventlog
spark.serializer org.apache.spark.serializer.KryoSerializer
spark.driver.memory 512m
3）spark-env.sh
export JAVA_HOME=/opt/lagou/servers/jdk1.8.0_231
export HADOOP_HOME=/opt/lagou/servers/hadoop-2.9.2
export HADOOP_CONF_DIR=/opt/lagou/servers/hadoop-2.9.2/etc/hadoop
export SPARK_DIST_CLASSPATH=$(/opt/lagou/servers/hadoop-2.9.2/bin/hadoop classpath)
export SPARK_MASTER_HOST=master
export SPARK_MASTER_PORT=7077
4）log4j.properties
cp log4j.properties.template log4j.properties
scp slaves slave2:/opt/lagou/servers/spark-2.4.5/conf/
scp spark-defaults.conf slave2:/opt/lagou/servers/spark-2.4.5/conf/
scp spark-env.sh slave2:/opt/lagou/servers/spark-2.4.5/conf
scp log4j.properties slave2:/opt/lagou/servers/spark-2.4.5/conf/
scp slaves slave3:/opt/lagou/servers/spark-2.4.5/conf/
scp spark-defaults.conf slave3:/opt/lagou/servers/spark-2.4.5/conf/
scp spark-env.sh slave3:/opt/lagou/servers/spark-2.4.5/conf
scp log4j.properties slave3:/opt/lagou/servers/spark-2.4.5/conf/
22.mysql安装
# 查询是否安装了
mariadb rpm -aq | grep mariadb删除mariadb。-e 删除指定的套件；--nodeps 不验证套件的相互关联性
rpm -e --nodeps mariadb-libs
安装依赖
yum install perl -y
yum install net-tools -y
yum install libaio
# 接压缩
tar xvf mysql-5.7.26-1.el7.x86_64.rpm-bundle.tar依次运行以下命令
rpm -ivh mysql-community-common-5.7.26-1.el7.x86_64.rpm
rpm -ivh mysql-community-libs-5.7.26-1.el7.x86_64.rpm
rpm -ivh mysql-community-client-5.7.26-1.el7.x86_64.rpm
rpm -ivh mysql-community-server-5.7.26-1.el7.x86_64.rpm