(1)cdh

Hadoop安装文档

CentOS系统配置

centos7关闭防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
关闭selinux
setenforce 0
vim /etc/selinux/config
SELINUX=disabled

centos7配置主机名
临时修改主机名
hostname master
永久修改主机名
vim /etc/hostname
master

修改网卡
TYPE=“Ethernet”
BOOTPROTO=“static”
NAME=“ens33”
DEVICE=“ens33”
IPADDR=“192.168.31.200”
ONBOOT=“yes”

重启网络
systemctl retart network.service

---

Java环境配置（三节点）

目前hadoop安装需要JDK8.0以上版本，同时JDK需要ORACLE官方JDK，不能使用OPENJDK
tar -xvf jdk-8u144-linux-x64.tar
cp -rf jdk-8u144-linux-x64.tar /usr/local/jdk

yum -y remove java

vim /etc/profile
export JAVA_HOME=/usr/local/jdk

mkdir /usr/share/java
cp /soft/mysql-connector-java-5.1.26-bin.jar /usr/share/java

mv /usr/share/java/mysql-connector-java-5.1.26-bin.jar /usr/share/java/mysql-connector-java.jar

---

创建HADOOP用户及相关目录（三节点）

创建hadoop用户
useradd hadoop
passwd hadoop

创建hadoop安装目录
mkdir /home/hadoop/install

---

配置互信

su - root
三节点生成公钥私钥
ssh-keygen
进入三节点ssh目录
cd /home/hadoop/.ssh
scp id_rsa.pub 192.168.2.99:~/.ssh/id_rsa.pub2
scp id_rsa.pub 192.168.2.99:~/.ssh/id_rsa.pub1
在master节点执行
cat id_rsa.pub1 id_rsa.pub id_rsa.pub2 >authorized_keys
分发authorized_keys到其他节点
修改authorized_keys权限
chmod 600 authorized_keys

---

安装mysql

解压MySQL的tar包
cd到mysql二进制包所在目录，然后使用tar命令进行解压

tar -xvf mysql-5.7.12-linux-glibc2.5-x86_64.tar.gz

复制相关目录
复制解压后的mysql到mysqlbase目录

cp -rf mysql-5.7.12-linux-glibc2.5-x86_64 /usr/local/mysql

创建MySQL用户
在linux中添加mysql用户

useradd mysql

创建MySQL数据目录
创建数据目录

mkdir -p /home/mysql3306/mysql3306
创建日志目录

mkdir -p /home/mysql3306/logs

修改相关目录权限
修改数据目录，日志目录，base目录权限

chown mysql:mysql -R /home/mysql3306
chown mysql:mysql -R /usr/local/mysql

配置文件
复制配置文件到/etc下

cp my3306.cnf /etc/my3306.cnf

初始化MySQL
MySQL5.7的初始化过程中，

/usr/local/mysql/bin/mysqld --defaults-file=/etc/my.cnf --initialize-insecure --basedir=/usr/local/mysql --datadir=/home/mysql3306/mysql3306/ --user=mysql &

启动MySQL

/usr/local/mysql/bin/mysqld_safe --defaults-file=/etc/my.cnf --datadir=/home/mysql3306/mysql3306/ --user=mysql &

进入MySQL
启动后密码为空

/usr/local/mysql/bin/mysql -S /tmp/mysql3306.sock

/opt/module/cm/cm-5.12.1/share/cmf/schema/scm_prepare_database.sh mysql scm -h192.168.10.102 -utemp -p000000 --scm-host 192.168.10.102 scm scm scm

select user,host,password from mysql.user;
grant all on . to temp@’%’ identified by ‘000000’ with grant option;
flush privileges;

安装完成后，简单初始化权限
因为安装完成后，没有详细的权限设置，所以我们需要手动的配置一下简单的权限

==================================================================

==================================================================

==================================================================

---

时间要一样

---

SSH 配置互信

---

创建用户cloudera-scm(所有节点)
useradd --system --home=/opt/cm-5.15.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment “Cloudera SCM User” cloudera-scm

---

拷贝JDBC JAR包(所有节点)
mkdir -p /usr/share/java
cp /home/hadoop/soft/mysql-connector-java-5.1.26-bin.jar /usr/share/java
mv /usr/share/java/mysql-connector-java-5.1.26-bin.jar /usr/share/java/mysql-connector-java.jar

---

在数据库里面赋权
grant all on . to root@’%’ identified by ‘root’;
flush privileges;

grant all privileges on . to temp@’%’ identified by ‘123456’ with grant option;
flush privileges;

create database hive ;

create database oozie ;

create database amon ;

create database hue;

修改 /etc/my.cnf
max_binlog_cache_size =8G

进入到 —>在mysql里面
set global max_binlog_cache_size =8388608000;

---

安装CDH5

master server/agent mysql
slave1 agent
slave2 agent

yum -y install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse portmap fuse-libs redhat-lsb

准备CDH包
安装CM server agent
mkdir /opt/
tar -xvf cloudera-manager-centos7-cm5.15.1_x86_64.tar.gz
mv /soft/cm-5.15.1 /opt/cm-5.15.1

修改配置文件
vim /opt/cm-5.15.1/etc/cloudera-scm-agent/config.ini

[General]

Hostname of the CM server.

server_host=192.168.2.10

Port that the CM server is listening on.

server_port=7182

拷贝到其他节点
scp -r /opt/cm-5.15.1 slave1:/opt/
scp -r /opt/cm-5.15.1 slave2:/opt/

---

执行脚本(在mysql里面构建SCM的存储集群信息的数据库)
/opt/cm-5.15.1/share/cmf/schema/scm_prepare_database.sh mysql scm -h 192.168.2.10 -utemp -p123456 --scm-host 192.168.2.10 scm scm scm

mysql 第一个mysql指定数据库类型
scm 库名
-h 192.168.2.10 数据库服务器
-uroot 用户名
-proot 密码
–scm-host manager节点
scm scm scm 默认写法，不用改

---

安装parcel

parcel
http://archive.cloudera.com/cdh5/parcels
CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel
CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel.sha1
manifest.json

server端
mkdir -p /opt/cloudera/parcel-repo
cp /soft/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel.sha1 manifest.json /opt/cloudera/parcel-repo/
mv /opt/cloudera/parcel-repo/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.15.1-1.cdh5.15.1.p0.4-el7.parcel.sha
将下载的parcel包均拷贝至主节点 /opt/cloudera/parcel-repo, 将*.sha1的包命名为*.sha

chown cloudera-scm:cloudera-scm /opt -R

agents端
mkdir -p /opt/cloudera/parcels
chown cloudera-scm:cloudera-scm /opt -R

---

启动

启动server端
/opt/cm-5.15.1/etc/init.d/cloudera-scm-server start

cd /opt/cm-5.15.1/log/cloudera-scm-server/
cat cloudera-scm-server.log
2018-10-17 07:14:03,795 INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.

启动agent端(在需要的机器上启动)
/opt/cm-5.15.1/etc/init.d/cloudera-scm-agent start

/opt/cm-5.15.1/etc/init.d/cloudera-scm-agent stop

访问端口
192.168.2.10:7180

账号密码

用户名/密码为admin/admin

配置集群

设置swap
echo “vm.swappiness = 10” >> /etc/sysctl.conf
scp /etc/sysctl.conf 192.168.2.11:/etc/

关闭大页面压缩
试过只设置defrag，但貌似个别节点还是会有警告，干脆全部设置，编辑/etc/rc.local，末尾添加如下命令
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
scp /etc/rc.local 192.168.2.11:/etc/

###############################
CM使用
1.监控管理集群主机
2.同意管理配置
3.统一维护集群

主机–host–服务器
机架–rack
集群–cluster
服务实例

/opt/module/cm/cm-5.12.1/etc/init.d/cloudera-scm-server start
/opt/module/cm/cm-5.12.1/etc/init.d/cloudera-scm-agent start

tail -f /opt/module/cm/cm-5.12.1/log/cloudera-scm-server/cloudera-scm-server.log

netstat -anp | grep 7180