物联网架构之CDH

1、常规初始化操作

三个节点都需要执行

hostnamectl set-hostname cdhmaster
yum -y install vim lrzsz net-tools

cat >>/etc/hosts<

192.168.180.210 cdhmaster
192.168.180.200 cdhslave01
192.168.180.190 cdhslave02
EOF

systemctl stop firewalld.service && systemctl disable firewalld.service

sed -i ‘/^SELINUX=/s/enforcing/disabled/’ /etc/selinux/config && setenforce 0

2、配置时间同步

cdhmaster:

yum install chrony -y

vim /etc/chrony.conf

server ntp6.aliyun.com iburst
allow 192.168.0.0/16

systemctl enable chronyd.service && systemctl restart chronyd.service

chronyc sources && chronyc -a makestep

cdhslave01/cdhslave02:

yum install chrony -y

vim /etc/chrony.conf
server cdhmaster iburst
systemctl enable chronyd.service && systemctl restart chronyd.service && chronyc sources

3、配置本机SSH免密码登录

ssh-keygen -t rsa

ssh-copy-id cdhmaster

ssh-copy-id cdhslave01

ssh-copy-id cdhslave02

4、安装jdk
三个节点都要执行

tar zxf jdk-8u171-linux-x64.tar.gz -C /opt/

mv /opt/jdk1.8.0_171/ /opt/jdk

cat >>/etc/profile< export JAVA_HOME=/opt/jdk
export PATH=$JAV{A}_{H}OME/bin:$PATH
EOF
source /etc/profile && java -version

5、禁用交换分区和透明大页
三个节点都要操作

sysctl -w vm.swappiness=0

echo “vm.swappiness=0” >> /etc/sysctl.conf

echo never > /sys/kernel/mm/transparent_hugepage/defrag

echo never > /sys/kernel/mm/transparent_hugepage/enabled

echo “echo never > /sys/kernel/mm/transparent_hugepage/defrag” >> /etc/rc.d/rc.local

echo “echo never > /sys/kernel/mm/transparent_hugepage/enabled” >> /etc/rc.d/rc.local

chmod +x /etc/rc.d/rc.local

6、安装数据库
只在主节点cdhmastar上操作

rpm -qa | grep mariadb

rpm -e --nodeps mariadb-libs-5.5.68-1.el7.x86_64

yum install -y libaio
useradd mysql -s /sbin/nologin

tar zvxf mysql-5.7.22-linux-glibc2.12-x86_64.tar.gz

mv mysql-5.7.22-linux-glibc2.12-x86_64 /usr/local/mysql

cd /usr/local/mysql/bin

./mysqld --initialize --user=mysql --datadir=/usr/local/mysql/data

自动生成数据库密码 IrszF>8vd(gy

cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld

vim /etc/my.cnf

[client]
port = 3306
socket = /usr/local/mysql/data/mysql.sock
[mysqld]
port = 3306
socket = /usr/local/mysql/data/mysql.sock
pid_file = /usr/local/mysql/data/mysql.pid
datadir = /usr/local/mysql/data
default_storage_engine = InnoDB
max_allowed_packet = 512M
max_connections = 2048
open_files_limit = 65535
skip-name-resolve
lower_case_table_names=1
character-set-server = utf8mb4
collation-server = utf8mb4_unicode_ci
init_connect=‘SET NAMES utf8mb4’
innodb_buffer_pool_size = 1024M
innodb_log_file_size = 2048M
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit = 0
key_buffer_size = 64M
log-error = /usr/local/mysql/data/mysql_error.log
log-bin = /usr/local/mysql/data/mysql-bin
slow_query_log = 1
slow_query_log_file = /usr/local/mysql/data/mysql_slow_query.log
long_query_time = 5
tmp_table_size = 32M
max_heap_table_size = 32M
query_cache_type = 0
query_cache_size = 0
server-id=1

/etc/init.d/mysqld start

ln -s /usr/local/mysql/bin/* /usr/sbin/

mysqladmin -uroot -p password ‘123456’

mysql -u root -p123456

alter user ‘root’@‘localhost’;

alter user ‘root’@‘localhost’ identified by ‘123456’;

create database hive default charset utf8 collate utf8_general_ci;

create database oozie default charset utf8 collate utf8_general_ci;

grant all privileges on . to ‘cdh’@‘localhost’ identified by ‘123456’ with grant option;

grant all privileges on . to ‘cdh’@‘%’ identified by ‘123456’ with grant option;

grant all privileges on . to ‘root’@‘127.0.0.1’ identified by ‘123456’ with grant option;

grant all privileges on . to ‘scm’@‘127.0.0.1’ identified by ‘123456’ with grant option;

flush privileges;

7、安装CDH
所有节点上安装依赖包

yum install -y psmisc libxslt libxslt-python perl

上传cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz、mysql-connector-java-5.1.46.tar.gz到服务器

tar zxf cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz -C /opt/

mv /opt/cm-5.14.0/ /opt/cm

tar zxf mysql-connector-java-5.1.46.tar.gz

cd mysql-connector-java-5.1.46

chmod +x mysql-connector-java-5.1.46-bin.jar

cp mysql-connector-java-5.1.46-bin.jar /opt/cm/share/cmf/lib/mysql-connector-java.jar

mkdir /usr/share/java

cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar

在三台机器上创建cloudera-scm用户

useradd --system --home=/opt/cm/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment “Cloudera SCM User” cloudera-scm

相关选项说明

–no-create-home 相当于-M,不创建用户的主目录

–shell 用户登录的shell,/bin/false禁止用户登录

–system 创建一个系统用户，在sys_uid_max中取ID

–comment 添加备注

/opt/cm/share/cmf/schema/scm_prepare_database.sh mysql cm -h localhost -uroot -p’123456’ scm ‘123456’
vim /opt/cm/etc/cloudera-scm-agent/config.ini

server_host=192.168.180.210

同步Cloudera Manager 数据到其他节点

scp -r /opt/cm cdhslave01:/opt/

scp -r /opt/cm cdhslave02:/opt/

上传manifest.json、CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel、CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1文件到cdhmaster服务器上

mv manifest.json CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1 /opt/cloudera/parcel-repo/

cd /opt/cloudera/parcel-repo/

mv CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1 CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha

vim /opt/cm/etc/cloudera-scm-server/db.properties

com.cloudera.cmf.db.useSSL=true

mkdir /var/lib/cloudera-scm-server

/opt/cm/etc/init.d/cloudera-scm-server start

/opt/cm/etc/init.d/cloudera-scm-agent start

netstat -anpt | grep 7180

在slave上只需启动代理服务

/opt/cm/etc/init.d/cloudera-scm-agent start

netstat -anpt | grep -E ‘19001|9000’

8、安装配置CDH 群集
访问CDH控制台
http://192.168.180.210:7180 默认的用户名和密码分别是admin 和admin。

三台机器都执行
上传mysql-connector-java-5.1.46.tar.gz到服务器

cd mysql-connector-java-5.1.46/

cp mysql-connector-java-5.1.46-bin.jar /opt/cloudera/parcels/CDH/lib/hive/lib/mysql-connector-java.jar

cp mysql-connector-java-5.1.46-bin.jar /opt/cloudera/parcels/CDH/lib/hadoop/lib/mysql-connector-java.jar

cp mysql-connector-java-5.1.46-bin.jar /var/lib/oozie/mysql-connector-java.jar

将上述三个文件复制到两台从节点

scp mysql-connector-java-5.1.46-bin.jar cdhslave01:/opt/cloudera/parcels/CDH/lib/hadoop/lib/mysql-connector-java.jar

scp mysql-connector-java-5.1.46-bin.jar cdhslave01:/opt/cloudera/parcels/CDH/lib/hive/lib/mysql-connector-java.jar

scp mysql-connector-java-5.1.46-bin.jar cdhslave01:/var/lib/oozie/mysql-connector-java.jar

scp mysql-connector-java-5.1.46-bin.jar cdhslave02:/opt/cloudera/parcels/CDH/lib/hive/lib/mysql-connector-java.jar

scp mysql-connector-java-5.1.46-bin.jar cdhslave02:/opt/cloudera/parcels/CDH/lib/hadoop/lib/mysql-connector-java.jar

scp mysql-connector-java-5.1.46-bin.jar cdhslave02:/var/lib/oozie/mysql-connector-java.jar

9、配置并添加Kafka 服务

先激活kafka

上传软件

mv KAFKA-1.2.0.jar /opt/cloudera/csd/

mv KAFKA-2.0.2-1.2.0.2.p0.5-el7.parcel /opt/cloudera/parcel-repo/

mv KAFKA-2.0.2-1.2.0.2.p0.5-el7.parcel.sha1 /opt/cloudera/parcel-repo/KAFKA-2.0.2-1.2.0.2.p0.5-el7.parcel.sha