CentOS-7.5安装Cloudera Manager伪分布式

一、下载相关文件

CentOS-7.5镜像

http://mirrors.huaweicloud.com/repository/centos/7/isos/x86_64/CentOS-7-x86_64-Minimal-1804.iso

Cloudera Manager 相关下载

http://archive.cloudera.com/cm5/cm/5/cloudera-manager-centos7-cm5.15.1_x86_64.tar.gz
http://archive.cloudera.com/cdh5/parcels/latest/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel
http://archive.cloudera.com/cdh5/parcels/latest/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel.sha1
http://archive.cloudera.com/cdh5/parcels/latest/manifest.json

JDK

请到oracle官网进行下载
本例中使用 jdk-8u181-linux-x64.tar.gz

二、安装虚拟机

安装过程略
PS: 安装过程中可设置root密码

三、环境准备

修改root密码，为方便操作，之后所有的命令均可使用root执行

sudo passwd root

设置网络及静态IP

ip addr
# 查看网卡名为 ens33
sudo vi /etc/sysconfig/network-scripts/ifcfg-ens33
# 修改以下内容
#----------------------------------#
BOOTPROTO=static
ONBOOT=yes
#----------------------------------#
# 文件最后添加以下内容
#----------------------------------#
GATEWAY=192.168.60.2
IPADDR=192.168.60.135
NETMASK=255.255.255.0
DNS1=8.8.8.8
#----------------------------------#

# 重启网络服务 
sudo service network restart 

安装vim

yum -y install vim

为方便操作，关闭防火墙、SELinux等

# 停止firewall
systemctl stop firewalld.service
# 禁止firewall开机启动
systemctl disable firewalld.service
# 查看默认防火墙状态
firewall-cmd --state 

vim /etc/selinux/config
# 修改以下内容
#----------------------------------#
SELINUX=disabled
#----------------------------------#

安装常用软件

sudo yum -y install lrzsz
sudo yum -y install unzip
sudo yum -y install psmisc
sudo yum -y install wget
sudo yum -y install libxslt-devel
sudo yum -y install epel-release
sudo yum -y install python-pip
sudo yum -y install net-tools

安装jdk8

# 查看CentOS自带JDK是否已安装。
yum list installed |grep java
# 卸载原java
yum -y remove java-1.7.0-openjdk*

mkdir /usr/java
tar zxvf ~/jdk-8u181-linux-x64.tar.gz -C /usr/java
vim /etc/profile
# 在文件末尾添加以下内容
#----------------------------------#
export JAVA_HOME=/usr/java/jdk1.8.0_181
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$JAVA_HOME/bin:$PATH
#----------------------------------#

# 使配置生效
source /etc/profile

python3.6

yum -y install python36
cd /usr/bin
rm -rf python
ln -s /usr/bin/python3.6 python
ln -s /usr/bin/python3.6 python3

#修复yum
vim /usr/bin/yum
vim /usr/libexec/urlgrabber-ext-down
#将 #!/usr/bin/python 修改为 #!/usr/bin/python2

ntp

yum -y install ntp
systemctl enable ntpd
systemctl start ntpd

# 配置是否为其他PC提供时间服务
vi /etc/ntp.conf
#----------------------------------#
restrict 192.168.60.0 mask 255.255.255.0 nomodify notrap
# 配置上游时间服务器为本地的ntpd Server服务器
server 192.168.60.135
#----------------------------------#

# 重启
systemctl restart ntpd

修改hostname

hostnamectl set-hostname hadoopm
# 使hostname生效
service network restart

修改hosts

sudo vim /etc/hosts

# 添加以下内容
#----------------------------------#
192.168.60.135 hadoopm
#----------------------------------#

免密码登录ssh

# 生成密钥
sudo ssh-keygen -t rsa
# 会在~/.ssh下面生成2个文件
# id_rsa，id_rsa.pub

# 在hadoopm上执行
sudo cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

安装mysql（mariaDB）

yum -y install mariadb mariadb-server
systemctl enable mariadb
systemctl start mariadb
# 此时mysql密码为空，设置mysql密码
mysql_secure_installation

mysql -uroot -pabc123_
# 可以使用远程登录root
#----------------------------------#
update user set host='%' where user='root' and host='localhost';
flush privileges;
#----------------------------------#

# 编辑/etc/my.cnf
vim /etc/my.cnf
# 在[mysqld]标签下添加下面内容
#----------------------------------#
default-storage-engine = innodb
innodb_file_per_table
max_connections = 4096
collation-server = utf8_general_ci
character-set-server = utf8
#----------------------------------#
# 编辑/etc/my.cnf.d/client.cnf
vim /etc/my.cnf.d/client.cnf
# 在[client]标签下添加下面内容
#----------------------------------#
default-character-set=utf8
#----------------------------------#
# 编辑/etc/my.cnf.d/mysql-clients.cnf
vim /etc/my.cnf.d/mysql-clients.cnf
# 在[mysql]标签下添加下面内容
#----------------------------------#
default-character-set=utf8
#----------------------------------#

# 重启
systemctl restart mariadb

#在mysql中执行，查看结果
#----------------------------------#
show variables like "%character%";
show variables like "%collation%";
#----------------------------------#

四、测试环境是否正常

略

五、安装Cloudera Manager

解压、复制文件

tar zxvf ~/cloudera-manager-centos7-cm5.15.1_x86_64.tar.gz -C /opt/
mkdir /opt/cloudera /opt/cloudera/parcels /opt/cloudera/parcel-repo
mv ~/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel.sha
mv ~/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel /opt/cloudera/parcel-repo/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel
mv ~/manifest.json /opt/cloudera/parcel-repo/manifest.json
rm -rf ~/cloudera-manager-centos7-cm5.15.1_x86_64.tar.gz

添加mysql 驱动

tar zxvf mysql-connector-java-5.1.46.tar.gz
mkdir /usr/share/java/
cp ~/mysql-connector-java-5.1.46/mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar
cp /usr/share/java/mysql-connector-java.jar /opt/cm-5.15.1/share/cmf/lib/mysql-connector-java.jar
rm -rf ~/mysql-connector*

根据Cloudera Manager的建议修改

echo 10 > /proc/sys/vm/swappiness
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

创建数据库

Service	Database	User
Cloudera Manager Server	scm	scm
Activity Monitor	amon	amon
Reports Manager	rman	rman
Hue	hue	hue
Hive Metastore Server	metastore	hive
Sentry Server	sentry	sentry
Cloudera Navigator Audit Server	nav	nav
Cloudera Navigator Metadata Server	navms	navms
Oozie	oozie	oozie

# 进入mysql
mysql -u root -p
CREATE DATABASE  DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON .* TO ''@'%' IDENTIFIED BY '';

# 以下为具体的创建语句
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'abc123_';
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'abc123_';
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'abc123_';
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'abc123_';
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'abc123_';
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON metastore.* TO 'hive'@'%' IDENTIFIED BY 'abc123_';
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'abc123_';
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'abc123_';
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'abc123_';

# SHOW DATABASES;
# SHOW GRANTS FOR ''@'%';

创建scm数据库

# 重新配置时，防止scm数据库已经存在而终止执行命令可加参数 -f
/opt/cm-5.15.1/share/cmf/schema/scm_prepare_database.sh mysql scm scm

配置agent

sudo vim /opt/cm-5.15.1/etc/cloudera-scm-agent/config.ini
#修改以下内容
server_host=hadoopm

创建cloudera-scm用户（各个节点）

useradd --system --home=/opt/cm-5.15.1/run/cloudera-scm-server --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

更改文件夹所有用户与用户组（各个节点）

mkdir /opt/cloudera /opt/cloudera/parcels /opt/cloudera/parcel-repo
sudo chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcels
sudo chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
sudo chown -R cloudera-scm:cloudera-scm /opt/cm-5.15.1

启动Cloudera Manager的Server和Agent

#在hadoopm执行
/opt/cm-5.15.1/etc/init.d/cloudera-scm-server start
#在各个节点执行
/opt/cm-5.15.1/etc/init.d/cloudera-scm-agent start

# 查看运行状态
/opt/cm-5.15.1/etc/init.d/cloudera-scm-agent status
/opt/cm-5.15.1/etc/init.d/cloudera-scm-server status

# 查看server日志
tail -f /opt/cm-5.15.1/log/cloudera-scm-server/cloudera-scm-server.log 
# 查看agent日志
tail -f /opt/cm-5.15.1/log/cloudera-scm-agent/cloudera-scm-agent.log 

六、图形化安装

访问管理界面

http://192.168.60.135:7180/
# 账号密码均为admin

可参考： https://www.jianshu.com/p/012739e132f6