CDH-6.3.2完整安装附带资源包

#########创建文件夹############

mkdir -p /opt/softwear
mkdir -p /opt/module

所有服务器都需要创建

############安装百度网盘##############
1、centos服务器安装百度云盘

pip install bypy

2、输入

bypy info

链接: https://pan.baidu.com/s/1ANDiUQIuZCgYiZOeFpFbxg 提取码: ouok
资源包4.11G有点儿大，完大数据别心痛。开一个月会员，下载速度嘎嘎快

3、复制连接在浏览器中使用百度账号登录
https://openapi.baidu.com/oauth/2.0/authorize?scope=basic+netdisk&redirect_uri=oob&response_type=code&client_id=q8WE4EpCsxxxx
4、将授权码复制到控制台在回车确认
629fa88xxxxx

5、将文件拷贝到我的应用数据 > bypy 文件夹中

6、执行下载命令

bypy downfile CDH.zip

############安装zip解压工具##############

yum -y install unzip

#############将IP映射到hosts中#################
vi /etc/hosts

172.20.146.111   hadoop101 hadoop101
172.20.146.112   hadoop102 hadoop102
172.20.146.113   hadoop103 hadoop103

############三台机器都设置免密登录,都执行以下代码##############

ssh-keygen -t rsa
ssh-copy-id hadoop101
ssh-copy-id hadoop102
ssh-copy-id hadoop103

############安装JDK##############
将解压的文件/opt/softwear/CDH/jdk-8u191-linux-x64.tar.gz
放到/usr/java文件夹中安装

mkdir -p /usr/java

三台服务器都创建文件夹

scp jdk-8u191-linux-x64.tar.gz root@hadoop102:/usr/java
scp jdk-8u191-linux-x64.tar.gz root@hadoop103:/usr/java

三台解压 安装

tar -zxvf jdk-8u191-linux-x64.tar.gz
vi /etc/profile
export JAVA_HOME=/usr/java/jdk1.8.0_191
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:${JAVA_HOME}/lib:${JRE_HOME}/lib:$CLASSPATH
export JAVA_PATH=${JAVA_HOME}/bin:${JRE_HOME}/bin
export PATH=$PATH:$JAVA_PATH/bin

source /etc/profile

############离线安装mysql##############
检查是否安装过mysql

rpm -qa | grep -i mysql
find / -name mysql

检查自带的mariadb并删除,只需要删除需要安装mysql服务器的

rpm -qa|grep mariadb
yum remove mariadb-libs

下载网盘资源包已经有mysql资源包
 mv mysql-5.7.38-el7-x86_64.tar.gz /opt/module/
 tar -zxvf mysql-5.7.38-el7-x86_64.tar.gz
 mv mysql-5.7.38-el7-x86_64 mysql

修改权限为mysql组、用户

 groupadd mysql
 useradd -g mysql mysql
 chown -R mysql /opt/module/mysql

查看添加的mysql用户情况

 id mysql

mkdir -p /opt/module/mysql/data

vi /etc/my.cnf

[mysqld]
#设置mysql的安装目录
basedir = /opt/module/mysql
#设置mysql数据库的数据存放目录
datadir = /opt/module/mysql/data
#设置端口
port = 3306

socket = /tmp/mysql.sock
#设置字符集
character-set-server=utf8
#日志存放目录
log-error = /opt/module/mysql/data/mysqld.log
pid-file = /opt/module/mysql/data/mysqld.pid
#允许时间类型的数据为零(去掉NO_ZERO_IN_DATE,NO_ZERO_DATE)
sql_mode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION
default-time_zone = '+8:00'
max_allowed_packet = 20M 

初始化mysql

yum -y install libaio
cd /opt/module/mysql
./bin/mysqld --initialize --user=mysql --basedir=/opt/module/mysql/ --datadir=/opt/module/mysql/data/

查看日志获得mysql初始化密码

cat /opt/module/mysql/data/mysqld.log

登录服务、修改密码

cd /opt/module/mysql/support-files
./mysql.server start
cd /opt/module/mysql/bin
./mysql -uroot -p

set password for root@localhost = password('xxxx');
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'xxxx' WITH GRANT OPTION;
flush privileges;
select User,authentication_string,Host from user;

启动mysql

cp /opt/module/mysql/support-files/mysql.server /etc/init.d/mysqld
chkconfig --add mysqld

############安装CM##############
会创建4个库

create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

创建文件夹 mkdir -p /usr/share/java
放入驱动包

mysql-connector-java-5.1.42.jar
mv mysql-connector-java-5.1.42.jar mysql-connector-java.jar
scp mysql-connector-java.jar root@hadoop102:/usr/share/java
scp mysql-connector-java.jar root@hadoop103:/usr/share/java

三台机器创建目录

mkdir -p /opt/cloudera-manager

mv cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm /opt/cloudera-manager
mv cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm /opt/cloudera-manager
mv cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm /opt/cloudera-manager
scp -r cloudera-manager/ root@hadoop102:/opt/
scp -r cloudera-manager/ root@hadoop103:/opt/

三台机器安装daemons

rpm -ivh cloudera-manager-daemons-6.3.1-1466458.el7.x86_64.rpm

三台机器安装agent

yum -y install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb portmap
yum -y install httpd mod_ssl openssl-devel python-psycopg2 MySQL-python
rpm -ivh cloudera-manager-agent-6.3.1-1466458.el7.x86_64.rpm

修改配置文件

vi /etc/cloudera-scm-agent/config.ini
server_host=hadoop101

只在hadoop101安装server

rpm -ivh cloudera-manager-server-6.3.1-1466458.el7.x86_64.rpm

初始化元数据库

vi /etc/cloudera-scm-server/db.properties
cd /opt/cloudera/cm/schema
./scm_prepare_database.sh mysql scm root

将离线包插件安装上

cd /opt/softwear/CDH/cdh/
mv CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel /opt/cloudera/parcel-repo/
mv CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha1 /opt/cloudera/parcel-repo/
mv manifest.json /opt/cloudera/parcel-repo/
mv /opt/cloudera/parcel-repo/CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel.sha

启动cdh

hadoop101
systemctl start cloudera-scm-server

hadoop101,hadoop102,hadoop103
systemctl start cloudera-scm-agent

查看日志
tail -f /var/log/coludera 里面有server agent都有
访问页面
http://hadoop101:7180/

优化：给hadoop NameNode启动高可用 HA
在配置项：搜索 dfs.permissions 取消勾选HDFS服务范围
点击操作 -》 启动high availability
JournalNode注意是用户同步日志用的，需要搭建奇数台服务器
设置dfs.journalnode.edits.dir = /dfs/jn

启动yarn高可用
首页 点击yarn -> 操作 -》 启动high availability