1. 下载安装包
1.1 CDH
$ wget http://archive.cloudera.com/cdh5/parcels/5.16.2/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel$ wget http://archive.cloudera.com/cdh5/parcels/5.16.2/CDH-5.16.2-1.cdh5.16.2.p0.8-el7.parcel.sha1$ wget http://archive.cloudera.com/cdh5/parcels/5.16.2/manifest.json1.2 Cloudera Mamager
$ wget http://archive.cloudera.com/cm5/cm/5/cloudera-manager-centos7-cm5.16.2_x86_64.tar.gz1.3 JDBC
$ wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.49.tar.gz2. 机器规划
| 系统 | IP | 主机名 |
|---|---|---|
| centos7.7 | 192.168.233.52 | mtr-kafka-1 |
| centos7.7 | 192.168.233.66 | mtr-kafka-2 |
| centos7.7 | 192.168.233.154 | mtr-kafka-3 |
| centos7.7 | 192.168.233.61 | mtr-kafka-4 |
| centos7.7 | 192.168.233.15 | mtr-kafka-5 |
3. 环境配置
3.1 关闭防火墙
所有机器执行
$ systemctl stop firewalld && systemctl disable firewalld3.2 设置免密
设置主从节点之间的免密登陆
3.3 Selinux
$ setenforce 0
$ vim /etc/selinux/config
SELINUX=disabled3.4 配置swappiness
$ sysctl -w vm.swappiness=10
$ echo vm.swappiness = 10 >> /etc/sysctl.conf
$ cat /proc/sys/vm/swappiness
103.5关闭透明大页面
$ echo never > /sys/kernel/mm/transparent_hugepage/defrag
$ echo never > /sys/kernel/mm/transparent_hugepage/enabled
$ vim /etc/rc.d/rc.local
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi
$ chmod +x /etc/rc.d/rc.local3.6 文件句柄
$ vim /etc/security/limits.conf
* soft nofile 100000
* hard nofile 1000003.7 时间服务器
$ yum install ntp ntpdate -y
$ systemctl start ntpd && systemctl enable ntpd3.8 操作系统基础软件
$ yum install -y chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb mod_ssl unzip lrzsz3.9 MySQL
$ wget https://cdn.mysql.com//archives/mysql-5.7/mysql-5.7.30-1.el7.x86_64.rpm-bundle.tar
tar -zvf mysql-5.7.30-1.el7.x86_64.rpm-bundle.tar
$ yum install mysql-community-common-5.7.30-1.el7.x86_64.rpm
$ yum install mysql-community-libs-5.7.30-1.el7.x86_64.rpm
$ yum install mysql-community-libs-compat-5.7.30-1.el7.x86_64.rpm
$ yum install mysql-community-client-5.7.30-1.el7.x86_64.rpm
$ yum install mysql-community-server-5.7.30-
1.el7.x86_64.rpm -y
$ systemctl start mysqld && systemctl enable mysqld$ mysql -uroot -h192.168.233.15 -p'Cdnhub010499@'
mysql> GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION;
mysql> GRANT ALL ON hive.* TO 'scm'@'%' WITH GRANT OPTION;
mysql> GRANT ALL ON am.* TO 'scm'@'%' WITH GRANT OPTION;
mysql> CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
mysql> CREATE DATABASE am DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
mysql> alter database hive character set latin1;MySQL配置远程root访问并修改密码，创建scm用户并创建远程登陆与密码。4. 安装Cloudera Manager
$ for i in {1..5};do ssh mtr-kafka-$i "useradd -r -d /data/cm-5.15.1/run/cloudera-scm-server -M -s /bin/false -c 'Cloudera SCM User' cloudera-scm";done
$ /data/cm-5.16.2/share/cmf/schema/scm_prepare_database.sh mysql -h 192.168.233.15 -uroot -p'Cdnhub010499@' --scm-host 192.168.233.52 scm scm 'Cdnhub010499@'
$ tar -zxvf mysql-connector-java-5.1.49.tar.gz
$ tar -zxvf cloudera-manager-centos7-cm5.16.2_x86_64.tar.gz -C /data/
$ for i in {2..5};do scp cloudera-manager-centos7-cm5.16.2_x86_64.tar.gz mtr-kafka-$i:/data/;done
$ for i in {2..5};do ssh mtr-kafka-$i "tar -zxvf ~/cloudera-manager-centos7-cm5.16.2_x86_64.tar.gz -C /data/"/;done
$ for i in {1..5};do scp mysql-connector-java-5.1.49/mysql-connector-java-5.1.49.jar mtr-kafka-$i:/data/cm-5.16.2/share/cmf/lib/;done
$ for i in {1..5};do ssh mtr-kafka-$i "cp ~/mysql-connector-java-5.1.49.jar /usr/share/java/mysql-connector-java.jar";done
$ for i in {1..5};do ssh mtr-kafka-$i "chmod -R cloudera-scm.cloudera-scm /data/cm-5.16.2";done
$ for i in {1..5};do ssh mtr-kafka-$i "chown -R cloudera-scm.cloudera-scm /data/cloudera";done
$ for i in {1..5};do ssh mtr-kafka-$i "chown -R cloudera-scm.cloudera-scm /opt/cloudera";done$ vim /data/cm-5.16.2/etc/cloudera-scm-server/db.properties
com.cloudera.cmf.db.type=mysql
com.cloudera.cmf.db.host=192.168.233.15
com.cloudera.cmf.db.name=scm
com.cloudera.cmf.db.user=scm
com.cloudera.cmf.db.password=Cdnhub010499@
com.cloudera.cmf.db.setupType=EXTERNAL$ for i in {1..5};do ssh mtr-kafka-$i "mkdir /opt/cloudera";done
$ for i in {1..5};do ssh mtr-kafka-$i "mkdir /opt/cloudera/parcels";done
$ for i in {1..5};do ssh mtr-kafka-$i "mkdir /opt/cloudera/parcel-repo";done
$ for i in {1..5};do ssh mtr-kafka-$i "mkdir /opt/cloudera/csd";done
$ for i in {1..5};do ssh mtr-kafka-$i "mkdir /opt/cloudera/parcel-cache";done
$ for i in {1..5};do ssh mtr-kafka-$i "chown cloudera-scm:cloudera-scm /data/cloudera";done
$ for i in {1..5};do ssh mtr-kafka-$i "sed -i "s/server_host=localhost/server_host=192.168.233.52/" /data/cm-5.16.2/etc/cloudera-scm-agent/config.ini";done$ /data/cm-5.16.2/etc/init.d/cloudera-scm-server start
$ for i in {1..5};do ssh mtr-kafka-$i "/data/cm-5.16.2/etc/init.d/cloudera-scm-agent start"; done登陆http://192.168.233.52:7180
默认用户与密码：admin
5. Spark2
由于CDH自集成的Spark为1.6.0版本，有点老了，所以需要安装2.0.0以上版本spark
$ cd /opt/cloudera/parcel-repo
$ mv manifest.json manifest_CDH5.16.2.json
$ wget http://archive.cloudera.com/spark2/parcels/2.4.0.cloudera2/manifest.json
$ wget http://archive.cloudera.com/spark2/parcels/2.4.0.cloudera2/SPARK2-2.4.0.cloudera2-1.cdh5.13.3.p0.1041012-el7.parcel.sha1
$ wget http://archive.cloudera.com/spark2/parcels/2.4.0.cloudera2/SPARK2-2.4.0.cloudera2-1.cdh5.13.3.p0.1041012-el7.parcel
$ mv SPARK2-2.4.0.cloudera2-1.cdh5.13.3.p0.1041012-el7.parcel.sha1 SPARK2-2.4.0.cloudera2-1.cdh5.13.3.p0.1041012-el7.parcel.sha
$ cd ../csd
$ wget http://archive.cloudera.com/spark2/csd/SPARK2_ON_YARN-2.4.0.cloudera2.jar登陆Cloudera Manager并进入Parcel管理界面
检查新Parcel，会出现SPARK2的选项，将他分配并激活即可
之后重启server与agent
$ /data/cm-5.16.2/etc/init.d/cloudera-scm-server restart
$ for i in {1..5};do ssh mtr-kafka-$i "/data/cm-5.16.2/etc/init.d/cloudera-scm-agent restart"; done之后再添加SPARK2服务