写于安装前
关于hadoop集群管理软件，我选择了cdh的免费版。本文参照官方文档、网上大神的帖子，加上自己几十次的搭建，踩了无数的坑总结而成。由于每个人的现场环境不同，参照本文档可能会出现报错情况，欢迎留言讨论交流。
1、基础环境
服务器配置:
24核48线程 64GB 600Gx2(raid1)
hadoop存储(4Tx10/每台机器)
目录-->/data1-->/data10
| 组建 | 版本 | 其他 | |
|---|---|---|---|
| OS | centos7.4.1708 （64位 3.1内核） | ||
| CDH | 5.13.0 | 安装目录:/opt/cm-5.13.0/ | |
| JDK | oracle jdk1.8.0_152 | 安装目录:/usr/java/jdk1.8 | |
| mysql | 5.7.20 |
| 角色 | 主机名 | IP | 备注 |
|---|---|---|---|
| Hadoop | cdh01 | 10.10.12.31 | |
| Hadoop | cdh02 | 10.10.12.32 | |
| Hadoop | cdh03 | 10.10.12.33 | |
| Hadoop | cdh04 | 10.10.12.34 | |
| Hadoop | cdh05 | 10.10.12.35 | |
| Mysql | 待定 | 10.10.12.38 | pwd |
备注：mysql服务器复用计算调度
2、安装依赖
首先安装mysql
sudo su - root
yum install vim lvm2 ssl_mod chkconfig python bind-utils psmisc libxslt zlib sqlite fuse fuse-libs redhat-lsb cyrus-sasl-plain cyrus-sasl-gssapi pstree httpd rsync -y
3、准备离线包
cloudera-manager-centos7-cm5.13.0_x86_64.tar.gz
CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha1
CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel
manifest.json
mysql-connector-java-5.1.44-bin.jar
oracle jdk 1.8.0_152
下载地址:
http://archive.cloudera.com/cm5/cm/5/
tttp://archive.cloudera.com/cdh5/parcels/latest/
4、 安装
4.1 所有节点
echo "*/20 * * * * /usr/sbin/ntpdate pool.ntp.org > /dev/null 2>&1" > /var/spool/cron/root
systemctl stop NetworkManager
systemctl disable NetworkManager
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
setenforce 0
systemctl stop firewalld
systemctl disable firewalld
cat >> /etc/hosts <优化hugepage
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
chmod +x /etc/rc.d/rc.local
cat >>/etc/rc.local< /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
EOF
echo "vm.swappiness = 0">> /etc/sysctl.conf
sysctl -p
4.2 配置主机名
###10.10.12.31
hostnamectl set-hostname cdh01
###10.10.12.32
hostnamectl set-hostname cdh02
###10.10.12.33
hostnamectl set-hostname cdh03
###10.10.12.34
hostnamectl set-hostname cdh04
###10.10.12.35
hostnamectl set-hostname cdh05
cat >> /etc/hosts <
10.10.12.32 cdh02
10.10.12.33 cdh03
10.10.12.34 cdh04
10.10.12.35 cdh05
END
4.3 配置ssh免密登陆
ssh-keygen -f $HOME/.ssh/id_rsa -t rsa -N ''
cat /root/.ssh/id_rsa.pub >/root/.ssh/authorized_keys
rsync -av /root/.ssh/* cdh01:/root/.ssh/
rsync -av /root/.ssh/* cdh02:/root/.ssh/
rsync -av /root/.ssh/* cdh03:/root/.ssh/
rsync -av /root/.ssh/* cdh04:/root/.ssh/
rsync -av /root/.ssh/* cdh05:/root/.ssh/
4.4 所有节点安装oracle jdk 1.8
4.4.1 主节点
yum install mod_ssl
mkdir -p /usr/java/jdk1.8
tar zxvf jdk-8u152-linux-x64.tar.gz -C /usr/java/
mv /usr/java/jdk1.8.0_152/* /usr/java/jdk1.8/
sed -i '$a export JAVA_HOME=/usr/java/jdk1.8/' /etc/profile
sed -i '$a export PATH=$JAVA_HOME/bin:$PATH' /etc/profile
source /etc/profile
java -version
echo $JAVA_HOME
rsync -av jdk-8u152-linux-x64.tar.gz cdh02:/usr/local/
rsync -av jdk-8u152-linux-x64.tar.gz cdh03:/usr/local/
rsync -av jdk-8u152-linux-x64.tar.gz cdh04:/usr/local/
rsync -av jdk-8u152-linux-x64.tar.gz cdh05:/usr/local/
4.4.2 其他节点
`cd /usr/local`
`mkdir -p /usr/java/jdk1.8`
`tar zxvf jdk-8u152-linux-x64.tar.gz -C /usr/java/`
`mv /usr/java/jdk1.8.0_152/* /usr/java/jdk1.8/`
`sed -i '$a export JAVA_HOME=/usr/java/jdk1.8/' /etc/profile`
`sed -i '$a export PATH=$JAVA_HOME/bin:$PATH' /etc/profile`
`source /etc/profile`
`java -version`
`echo $JAVA_HOME`
4.4.3 安装mysql(两个实例3306 3307)
rpm -aq|grep mariadb|xargs rpm -e --nodeps
rpm -ivh http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm
yum install mysql-server
vi /etc/my.cnf
binlog_format=mixed
systemctl restart mysqld
mysql授权
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
grant all privileges on *.* to root@"localhost" identified by pwd;
grant all privileges on *.* to root@"%" identified by pwd;
grant all privileges on hive.* to hive@"localhost" identified by pwd;
grant all privileges on hive.* to hive@"%" identified by pwd;
grant all privileges on hue.* to hue@"localhost" identified by pwd;
grant all privileges on hue.* to hue@"%" identified by pwd;
grant all privileges on oozie.* to oozie@"localhost" identified by pwd;
grant all privileges on oozie.* to oozie@"%" identified by pwd;
grant all privileges on cm.* to scm@"%" identified by pwd;
主节点
mkdir -p /var/lib/cloudera-scm-server/
cd /root/app/
tar -zxvf cloudera-manager-centos7-cm5.13.0_x86_64.tar.gz -C /opt
cp mysql-connector-java-5.1.44-bin.jar /opt/cm-5.13.0/share/cmf/lib/
/opt/cm-5.13.0/share/cmf/schema/scm_prepare_database.sh mysql cm -hhost -uroot -ppwd--scm-host cdh01 scm scm scm
vi /opt/cm-5.13.0/etc/cloudera-scm-agent/config.ini
server_host=cdh01
rsync -av /opt/cm-5.13.0 root@cdh02:/opt/
rsync -av /opt/cm-5.13.0 root@cdh03:/opt/
rsync -av /opt/cm-5.13.0 root@cdh04:/opt/
rsync -av /opt/cm-5.13.0 root@cdh05:/opt/
所有节点
useradd --system --home=/opt/cm-5.13.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
主节点
mkdir -p /opt/cloudera/parcel-repo/
cp /root/app/CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel /opt/cloudera/parcel-repo/
cp /root/app/CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha1 /opt/cloudera/parcel-repo/
cp /root/app/manifest.json /opt/cloudera/parcel-repo/
mv /opt/cloudera/parcel-repo/CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha
mkdir -p /usr/share/java/
cp /root/app/mysql-connector-java-5.1.44-bin.jar /usr/share/java/mysql-connector-java.jar
#cp /root/app/mysql-connector-java-5.1.44-bin.jar /opt/cloudera/parcels/CDH-5.13.0-1.cdh5.13.0.p0.29/lib/hive/lib/
/opt/cm-5.13.0/etc/init.d/cloudera-scm-server start
tailf /opt/cm-5.13.0/log/cloudera-scm-server/cloudera-scm-server.log
Table 'cm.CM_VERSION' doesn't exist
所有节点
/opt/cm-5.13.0/etc/init.d/cloudera-scm-agent start
/opt/cm-5.13.0/etc/init.d/cloudera-scm-agent restart
tailf /opt/cm-5.13.0/log/cloudera-scm-agent/cloudera-scm-agent.log
打开浏览器
http://10.10.12.31:7180
usermod -s /bin/bash hue
echo "hue ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/hue
chmod 0440 /etc/sudoers.d/hue
vi /opt/cm-5.13.0/lib64/cmf/service/hue/httpd.sh
add after line No. 129
sudo su - root -c "chown hue: $CLOUDERA_HTTPD_CONF_DIR/*.conf"
5、启用HDFS 的 High Availability
5.1 在HDFS上点击"启用High Availability"
5.2停止hive
更新 Hive Metastore NameNode
写于安装后
至此，hadoop集群能够完整运行。后续会写上spark、hive等组件的升级过程。本文侵删。