centos安装CDH6.3

一. CentOS更改yum源与更新系统

yum install -y wget

mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup

cd /etc/yum.repos.d/

wget http://mirrors.163.com/.help/CentOS6-Base-163.repo

yum makecache

yum -y update

yum install -y perl perl-devel vim lrzsz net-tools autoconf  make gcc-c++ cmake bison-devel ncurses-devel libaio libaio-devel

二. 集群规划

虚拟主机名	IP
cdh-master	192.168.2.235
cdh-slave1	192.168.2.236
cdh-slave2	192.168.2.237

  2.1 修改集群主机名

vim /etc/hostname

2.2 修改host文件

vim /etc/hosts

192.168.2.235 cdh-master
192.168.2.236 cdh-slave1
192.168.2.237 cdh-slave2

#重启
shutdown -r now

2.3 关闭防火墙和清空规则

systemctl stop firewalld
systemctl disable firewalld
iptables -F

2.4 selinux关闭

vim /etc/selinux/config


#修改内容
selinux=disabled

2.5 同步服务器时间

      修改主节点

vim /etc/ntp.conf

#添加配置
server 127.127.1.0 iburst local clock
restrict 192.168.1.0 mask 255.255.255.0 nomodify notrap

systemctl start ntpd
systemctl status ntpd
ntpq -p

     修改从节点

systemctl stop ntpd
systemctl disable ntpd
ntpdate cdh-master
crontab -e

#添加参数

 # 分 时 日 月 周，每天00:00执行该脚本
 00 00 * * * /usr/sbin/ntpdate cdh-master

三. 安装JDK

jdk1.8-rpm文件下载地址

rpm -ivh  oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm

设置环境变量

vim /etc/profile

# /etc/profile

# System wide environment and startup programs, for login setup
# Functions and aliases go in /etc/bashrc

# It's NOT a good idea to change this file unless you know what you
# are doing. It's much better to create a custom.sh shell script in
# /etc/profile.d/ to make custom changes to your environment, as this
# will prevent the need for merging in future updates.

pathmunge () {
    case ":${PATH}:" in
        *:"$1":*)
            ;;
        *)
            if [ "$2" = "after" ] ; then
                PATH=$PATH:$1
            else
                PATH=$1:$PATH
            fi
    esac
}


export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
export PATH=$PATH:$JAVA_HOME/bin
export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib


if [ -x /usr/bin/id ]; then
    if [ -z "$EUID" ]; then
        # ksh workaround
        EUID=`/usr/bin/id -u`
        UID=`/usr/bin/id -ru`
    fi
    USER="`/usr/bin/id -un`"
    LOGNAME=$USER
    MAIL="/var/spool/mail/$USER"
fi

# Path manipulation
if [ "$EUID" = "0" ]; then
    pathmunge /usr/sbin
    pathmunge /usr/local/sbin
else
    pathmunge /usr/local/sbin after
    pathmunge /usr/sbin after
fi

HOSTNAME=`/usr/bin/hostname 2>/dev/null`
HISTSIZE=1000
if [ "$HISTCONTROL" = "ignorespace" ] ; then
    export HISTCONTROL=ignoreboth
else
    export HISTCONTROL=ignoredups
fi

export PATH USER LOGNAME MAIL HOSTNAME HISTSIZE HISTCONTROL

# By default, we want umask to get set. This sets it for login shell
# Current threshold for system reserved uid/gids is 200
# You could check uidgid reservation validity in
# /usr/share/doc/setup-*/uidgid file
if [ $UID -gt 199 ] && [ "`/usr/bin/id -gn`" = "`/usr/bin/id -un`" ]; then
    umask 002
else
    umask 022
fi

for i in /etc/profile.d/*.sh /etc/profile.d/sh.local ; do
    if [ -r "$i" ]; then
        if [ "${-#*i}" != "$-" ]; then 
            . "$i"
        else
            . "$i" >/dev/null
        fi
    fi
done

unset i
unset -f pathmunge

source /etc/profile

四. master节点安装mysql5.6

4.1  卸载自带mariadb

rpm -qa | grep -i mariadb

rpm -e  --nodeps mariadb-libs-5.5.65-1.el7.x86_64

4.2 安装mysql

wget http://repo.mysql.com/mysql-community-release-el6-5.noarch.rpm

rpm -ivh mysql-community-release-el7-5.noarch.rpm

#安装mysql
yum install mysql-community-server

#启动mysql
service mysqld start

4.3 设置root用户权限及密码

mysql -u root

use mysql;
#设置密码
update user set password=PASSWORD("这里输入root用户密码") where User='root';
#设置远程访问权限
GRANT ALL PRIVILEGES ON *.* TO 'your username'@'%' IDENTIFIED BY 'your password';
flush privileges;

4.4 启动mysql服务

#启动 mysql
systemctl start mysqld.service
#重启 mysql
systemctl restart mysqld.service 
#停止 mysql
systemctl stop mysqld.service
#设置 mysql 开机启动
systemctl enable mysqld.service

mysql常用文件路径：

/etc/my.cnf 　　这是mysql的主配置文件

/var/lib/mysql 　　mysql数据库的数据库文件存放位置

/var/logs/mysqld.log　　数据库的日志输出存放位置

五. 创建CDH所需要的的数据库

grant all on scm.* to 'scm'@'%' identified by 'scm';
create database amon default character set utf8 default collate utf8_general_ci;
grant all on amon.* to 'amon'@'%' identified by 'amon';
create database rman default character set utf8 default collate utf8_general_ci;
grant all on rman.* to 'rman'@'%' identified by 'rman';
create database hue default character set utf8 default collate utf8_general_ci;
grant all on hue.* to 'hue'@'%' identified by 'hue';
create database metastore default character set utf8 default collate utf8_general_ci;
grant all on metastore.* to 'hive'@'%' identified by 'hive';
create database sentry default character set utf8 default collate utf8_general_ci;
grant all on sentry.* to 'sentry'@'%' identified by 'sentry';
create database nav default character set utf8 default collate utf8_general_ci;
grant all on nav.* to 'nav'@'%' identified by 'nav';
create database oozie default character set utf8 default collate utf8_general_ci;
grant all on oozie.* to 'oozie'@'%' identified by 'oozie';
flush privileges;

六. 设置集群间SSH免密登录

 6.1  修改master服务器SSH 配置文件

vim /etc/ssh/sshd_config

将PermitRootLogin no 或者 PermitRootLogin without-password 修改成 PermitRootLogin yes

6.2 生成秘钥对

ssh-keygen -t rsa

-t参数表示类型，这里选择rsa。选择保存位置的时候直接回车，使用默认的/root/.ssh/id_rsa。提示输入密码的时候，直接回车

6.3 将秘钥拷贝到.ssh目录下的authorized_keys

cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys

6.4 将公钥复制到两台slave服务器

ssh-copy-id -i ~/.ssh/id_rsa.pub [email protected]
ssh-copy-id -i ~/.ssh/id_rsa.pub [email protected]

设置完成后，同理设置slave1，slave2两台服务器

七. 创建CDH文件夹，并上传CDH相关文件

| --/opt
      |--/cloudera
                 |--/parcels
                 |--/parcel-repo

八.  rpm包部署CM

tar -xzvf cm6.3.1-redhat7.tar.gz -C /opt

cd /opt/cm6.3.1/RPMS/x86_64/

rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm --nodeps --force

rpm -ivh cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm --nodeps --force

rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm --nodeps --force

 修改cm-server配置文件

vim /etc/cloudera-scm-server/db.properties


#修改配置文件内容
com.cloudera.cmf.db.type=mysql

com.cloudera.cmf.db.host=cdh-master

com.cloudera.cmf.db.name=scm

com.cloudera.cmf.db.user=scm

com.cloudera.cmf.db.password=scm

com.cloudera.cmf.db.setupType=EXTERNAL

添加scm数据库用户

/opt/cloudera/cm/schema/scm_prepare_database.sh mysql -uroot -p你的MySQL密码 scm scm scm --force

启动cm-server

systemctl start cloudera-scm-server.service
#停止
systemctl start cloudera-scm-server.service
#重启
systemctl restart cloudera-scm-server.service
#查看服务启动状态
systemctl status cloudera-scm-server.service

查看cm-server日志

tail -f cat /var/log/cloudera-scm-server/cloudera-scm-server.log

修改cm-agent配置

vim /etc/cloudera-scm-agent/config.ini

#修改的配置文件内容
server_host=cdh-master

将rpm文件发送至从节点

#cdh-slave1
scp /opt/tools/cm6.3.1/RPMS/x86_64/cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm cdh-slave1:/opt/tools

scp /opt/tools/cm6.3.1/RPMS/x86_64/cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm cdh-slave1:/opt/tools

#cdh-slave2
scp /opt/tools/cm6.3.1/RPMS/x86_64/cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm cdh-slave2:/opt/tools

scp /opt/tools/cm6.3.1/RPMS/x86_64/cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm cdh-slave2:/opt/tools

从节点安装jdk、cloudera-manager-agent、cloudera-manager-daemons。安装流程参照主节点安装流程。

启动主、从节点的cloudera-manager-agent

systemctl start cloudera-scm-agent.service

#查看启动状态
systemctl status cloudera-scm-agent.service

查询端口启用状态

netstat -aptn

cm-server访问地址:   http://192.168.2.235:7180 

九. 选择安装CDH服务

 

选择cdh版本

设置cdh文件目录

安装

服务选择