CDH 5.9.3 集群配置



-----------------------------------------集群规划------------------------------------------
    hostname    ip        mac            硬盘    内存    
    cMaster        192.168.1.100    00:50:56:3D:97:A5        25GB    3G
    cSlave01        192.168.1.101    00:50:56:28:C5:C9        20GB    2G
    cSlave02        192.168.1.102    00:50:56:2E:8F:A7        20GB    2G

    文件名                    上传目录    目标目录                
    cloudera-manager-el6-cm5.9.3_x86_64.tar.gz    /opt/    /opt/cloudera-manager
    CDH-5.9.3-1.cdh5.9.3.p0.4-el6.parcel        
    CDH-5.9.3-1.cdh5.9.3.p0.4-el6.parcel.sha1    改名:CDH-5.9.3-1.cdh5.9.3.p0.4-el6.parcel.sha
    manifest.json

    目录            用户        属性
    opt/cloudera-manager    root        包含cloudera-manager-el6-cm5.9.3_x86_64.tar.gz解压文件
    /opt/cloudera/parcel-repo    cloudera-scm    存放CDH-5.9.3-1.cdh5.9.3.p0.4-el6.parcel
                        CDH-5.9.3-1.cdh5.9.3.p0.4-el6.parcel.sha
                        manifest.json
    /opt/cloudera/parcels    cloudera-scm    CM代理目录----------hosts文件-----hosts2.txt--------
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.100    cMaster
192.168.1.101    cSlave01
192.168.1.102    cSlave02
_______________________________________基础配置________________________________________________
1-虚拟机/安装VMware tool、双击打开VMwareTool tar包/解压到文件系统tmp
./tmp/vmware-tools-distrib/vmware-install.pl
reboot
1.网络桥接 设置物理地址
vi /etc/sysconfig/network
2.面板删连接配置,只剩System eth0
rm -rf /etc/udev/rules.d/70-persistent-net.rules 
cp /mnt/hgfs/setup/hosts /etc/hosts
reboot
vi /etc/udev/rules.d/70-persistent-net.rules
vi /etc/sysconfig/network-scripts/ifcfg-eth0    物理地址大写
----------------------------------------------------------ifcfg-eth0文件---------------------
DEVICE="eth0"
BOOTPROTO=none
IPV6INIT="yes"
NM_CONTROLLED="yes"
ONBOOT="yes"
TYPE="Ethernet"
IPADDR=192.168.1.120
PREFIX=24
GATEWAY=192.168.1.1
DNS1=192.168.1.1
DEFROUTE=yes
IPV4_FAILURE_FATAL=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
NAME="System eth0"
HWADDR=00:50:56:2A:C2:8D
IPV6_PEERDNS=yes
IPV6_PEERROUTES=yes


----------------------------------------------------------------------
service iptables stop
chkconfig iptables off

service network restart

------------------------ jdk 安装 -----------------root-----------
 cp /mnt/hgfs/setup/jdk-8u211-linux-x64.rpm /opt/
rpm -ivh /mnt/hgfs/setup/jdk-8u211-linux-x64.rpm 
which java
ll /usr/java/jdk1.8.0_161/bin/java        Java路径:/usr/java/jdk1.8.0_161
vi /etc/profile

----------------------profile 文件---------------
export JAVA_HOME=/usr/java/jdk1.8.0_161
export JRE_HOME=$JAVA/jre
export PATH=$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
-----------------------------------------------
***********************卸载JDK**********************************************
rpm -qa|grep jdk
---看到:jdk-1.6.0_22-fcs
rpm -e --nodeps jdk-1.6.0_22-fcs   
***************************************************************************

source /etc/profile
java -versio


2、SSH免密钥登录
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
cd /root/.ssh
scp id_dsa.pub node2:/tmp/
cat /tmp/id_dsa.pub >>authorized_keys
scp id_dsa.pub node1:/tmp/
cat /tmp/id_dsa.pub >>authorized_keys


3、防火墙关闭
service iptables stop
chkconfig iptables off
chkconfig --list iptables
service iptables status



4、SELINUX关闭
setenforce 0
vi /etc/selinux/config(SELINUX=disabled)


6、安装NTP
设置开机启动 chkconfig ntpd on
设置时间同步
ntpdate-u s2m.time.edu.cn

yum install ntp -y
date
chkconfig ntpd on
service ntpd start
ntpdate -u s2m.time.edu.cn
ntpdate -u node1

_____________________________________数据库配置___________________________________________
--GRANT ALL PRIVILEGES ON*.*TO'root'@"%'IDENTIFIED BY"123'WITH GRANT OPTION; --flush privileges yum install mysql-server service mysqld start chkconfig mysqld on chkconfig --list mysqld #2~5即为成功on mysql show databases; use mysql show tables; select host,user from user; GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY '123' WITH GRANT OPTION; delete from user where host !='%'; flush privileges; quit mysql -u root -p quit 8、下载第三方依赖包 yum install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb -y ___________________________________CM配置____________________________________________ CM安装 -1、安装Cloudera Manager Server、Agent ·mkdir/opt/cloudera-manager ·tar xvzf cloudera-manager*.tar.gz-C/opt/cloudera-manager -2、创建用户cloudera-scm useradd --system --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm -3、配置CM Agent ·修改文件/opt/cloudera-manager/cm-5.4.3/etc/cloudera-scm-agent/config.ini中 server_host以及server_port [root@node1 /]# scp -r /opt/cloudera-manager-el6-cm5.12.1_x86_64.tar.gz node2:/opt/ mkdir -p /opt/cloudera-manager #都执行 [root@node1 /]# tar -zxvf /opt/cloudera-manager-el6-cm5.12.1_x86_64.tar.gz -C /opt/cloudera-manager/ [root@node2 /]# tar -zxvf /opt/cloudera-manager-el6-cm5.12.1_x86_64.tar.gz -C /opt/cloudera-manager/ [root@node1 /]# useradd --system --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm [root@node2 /]# useradd --system --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm [root@node1 /]# cd /opt/cloudera-manager/ [root@node1 cloudera-manager]# ls [root@node1 cloudera-manager]# cd cm-5.12.1/ [root@node1 cm-5.12.1]# ls [root@node1 cm-5.12.1]# cd etc/ [root@node1 etc]# ls [root@node1 etc]# cd cloudera-scm-agent/ [root@node1 cloudera-scm-agent]# ls [root@node1 cloudera-scm-agent]# vi config.ini server_host=node1 [root@node2 /]# cd /opt/cloudera-manager/cm-5.12.1/etc/cloudera-scm-agent/ [root@node2 cloudera-scm-agent]# ls [root@node2 cloudera-scm-agent]# vi config.ini server_host=node1 -4、配置CM Server数据库 ·拷贝mysql jar文件到目录/usr/share/java/ ·注意意ar包名称要修改为mysql-connector-java.jar ·grant all on**to'temp'@%identified by temp'with grant option; ·cd/opt/cloudera-manager/cm-5.4.3/share/cmf/schema/ ·./scm_prepare_database.sh mysql temp-h node1-utemp-ptemp--scm-host node1 scm scm scm ·格式:数据库类型、数据库、数据库服务器、用户名、密码、cm server服务器 -5、创建Parcel目录 ·Server节点 -mkdir-p/opt/cloudera/parcel-repo -chown cloudera-scm:cloudera-scm/opt/cloudera/parcel-repo ·Agent节点 -mkdir-p/opt/cloudera/parcels -chown cloudera-scm:cloudera-scm/opt/cloudera/parcels [root@node1 cloudera-scm-agent]# cd /usr/share/ [root@node1 share]# ls [root@node1 share]# mkdir -p java [root@node1 share]# cd java [root@node1 java]# ls [root@node1 java]# cp /mnt/hgfs/setup/mysql-connector-java-5.1.46/mysql-connector-java-5.1.46/mysql-connector-java-5.1.46-bin.jar ./ [root@node1 java]# ls [root@node1 java]# mv mysql-connector-java-5.1.46-bin.jar mysql-connector-java.jar [root@node1 java]# ls 利用navicat连接192.168.1.101的MySQL,建立查询,输入: grant all on *.* to 'temp'@'%' identified by 'temp' with grant option; [root@node1 java]# cd /opt/cloudera-manager/cm-5.12.1/share/cmf/schema/ [root@node1 schema]# ls [root@node1 schema]# ./scm_prepare_database.sh mysql temp -h node1 -utemp -ptemp --scm-host node1 scm scm scm 当出现All done, your SCM database is configured correctly! 表示成功! [root@node1 schema]# mkdir -p /opt/cloudera/parcel-repo [root@node1 schema]# chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo [root@node1 schema]# mkdir -p /opt/cloudera/parcels [root@node1 schema]# chown cloudera-scm:cloudera-scm /opt/cloudera/parcels [root@node2 cloudera-scm-agent]# mkdir -p /opt/cloudera/parcels [root@node2 cloudera-scm-agent]# chown cloudera-scm:cloudera-scm /opt/cloudera/parcels -6、制作CDH本地源 ·下载好文件CDH-5.4.0-1.cdh5.4.0.p0.27-el6.parcel以及manifestjson,将这两个文件放到server节点的 /opt/cloudera/parcel-repo下。 ·打开manifest.json文件,里面是json格式的配置,找到与下载版本相对应的hash码,新建文件,文件名与你的parel包名一致,并加上.sha后缀,将hash码复制到文件中保存。 -7、启动CM Server、Agent ·cd/opt/cloudera-manager/cm-5.4.3/etc/init.d/ ·./cloudera-scm-server start -Sever首次启动会自动创建表以及数据,不要立即关闭或重启,否则需要删除所有表及数据重新安装 ·./cloudera-scm-agent start ·访问:http://ManagerHost:7180, ·用户名、密码:admin ·若可以访问,则CM安装成功。 [root@node1 opt]# cd /opt/cloudera/parcel-repo/ [root@node1 parcel-repo]# ll [root@node1 parcel-repo]# mv /opt/CDH-5.16.1-1.cdh5.16.1.p0.3-el6.parcel* ./ [root@node1 parcel-repo]# ll [root@node1 parcel-repo]# mv /opt/manifest.json ./ [root@node1 parcel-repo]# ll [root@node1 parcel-repo]# mv CDH-5.16.1-1.cdh5.16.1.p0.3-el6.parcel.sha1 CDH-5.16.1-1.cdh5.16.1.p0.3-el6.parcel.sha [root@node1 parcel-repo]# ll 【主节点】 检查主机正确性时出现 “已启用“透明大页面”,它可能会导致重大的性能问题。” 的警告,进行如下设定 echo never > /sys/kernel/mm/transparent_hugepage/enabled echo never > /sys/kernel/mm/transparent_hugepage/defrag # vi /etc/rc.local ------------rc.local---------------- echo never > /sys/kernel/mm/transparent_hugepage/enabled echo never > /sys/kernel/mm/transparent_hugepage/defrag ---------------------- Cloudera 建议将 /proc/sys/vm/swappiness 设置为 0。 echo 0 > /proc/sys/vm/swappiness [root@node1 parcel-repo]# cd /opt/cloudera-manager/cm-5.12.1/etc/init.d/ [root@node1 init.d]# ls [root@node1 init.d]# ./cloudera-scm-server start #开启 [root@node1 init.d]# ./cloudera-scm-server stop #关闭 [root@node2 init.d]# ./cloudera-scm-agent start #开启 [root@node1 init.d]# cd ../.. [root@node1 cm-5.12.1]# ls [root@node1 cm-5.12.1]# cd log/ [root@node1 log]# ls [root@node1 log]# cd cloudera-scm-server/ [root@node1 cloudera-scm-server]# ls [root@node1 cloudera-scm-server]# tail -f cloudera-scm-server.log cd ../.. cd log/ cd cloudera-scm-server/ tail -f cloudera-scm-server.log 出现:Started Jetty server 启动成功 ______________________________________________主节点所有命令____________________________________ cd / /tmp/vmware-tools-distrib/vmware-install.pl reboot cd / vi /etc/udev/rules.d/70-persistent-net.rules vi /etc/sysconfig/network-scripts/ifcfg-eth0 service iptables stop chkconfig iptables off service network restart ifconfig ping www.baidu.com cp /mnt/hgfs/setup/hosts2.txt /etc/hosts vi /etc/hosts rpm -ivh /mnt/hgfs/setup/jdk-7u80-linux-x64.rpm vi /etc/profile source /etc/profile java -versio java java -version ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys service iptables stop chkconfig iptables off chkconfig --list iptables service iptables status setenforce 0 vi /etc/selinux/config yum install ntp -y date chkconfig ntpd on service ntpd start ntpdate-u s2m.time.edu.cn ntpdate -u s2m.time.edu.cn yum install mysql-server service mysqld start chkconfig mysqld on chkconfig --list mysqld mysql mysql -u root -p yum install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb -y mkdir -p /opt/cloudera-manager tar -zxvf /opt/cloudera-manager-el6-cm5.9.3_x86_64.tar.gz -C /opt/cloudera-manager/ useradd --system --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm cd /opt/cloudera-manager/cm-5.12.1/etc/cloudera-scm-agent/ cd /opt/cloudera-manager ll cd /opt/cloudera-manager/cm-5.9.3/etc/cloudera-scm-agent/ ll vi config.ini cd /usr/share/ ll cd java ll cp /mnt/hgfs/setup/mysql-connector-java-5.1.46-bin.jar ./ ll mv mysql-connector-java-5.1.46-bin.jar mysql-connector-java.jar ll ./opt/cloudera-manager/cm-5.9.3/share/cmf/schema/scm_prepare_database.sh mysql temp -h cMaster -utemp -ptemp --scm-host cMaster scm scm scm cd /opt/cloudera-manager/cm-5.9.3/share/cmf/schema/ ll ./scm_prepare_database.sh mysql temp -h cMaster -utemp -ptemp --scm-host cMaster scm scm scm mkdir -p /opt/cloudera/parcel-repo chown cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo mkdir -p /opt/cloudera/parcels chown cloudera-scm:cloudera-scm /opt/cloudera/parcels cd /opt/cloudera/parcel-repo/ ll cd .. ll cd parcel-repo/ ll mv CDH-5.9.3-1.cdh5.9.3.p0.4-el6.parcel.sha1 CDH-5.9.3-1.cdh5.9.3.p0.4-el6.parcel.sha ll cd /opt/cloudera-manager/cm-5.12.1/etc/init.d/ cd /opt/cloudera-manager/cm-5.9.3/etc/init.d/ ll echo never > /sys/kernel/mm/transparent_hugepage/enabled echo never > /sys/kernel/mm/transparent_hugepage/defrag vi /etc/rc.local echo 0 > /proc/sys/vm/swappiness ./cloudera-scm-server start ./cloudera-scm-agent start cd ../.. cd log/ cd cloudera-scm-server/ tail -f cloudera-scm-server.log history ____________________________________________从节点命令表__________________________________ 1 cd / 2 ./tmp/vmware-tools-distrib/vmware-install.pl 3 reboot 4 vi /etc/sysconfig/network 5 vi /etc/udev/rules.d/70-persistent-net.rules 6 vi /etc/sysconfig/network-scripts/ifcfg-eth0 7 vi /etc/udev/rules.d/70-persistent-net.rules 8 vi /etc/sysconfig/network-scripts/ifcfg-eth0 9 service iptables stop 10 chkconfig iptables off 11 service network restart 12 ifconfig 13 rpm -ivh /mnt/hgfs/setup/jdk-7u80-linux-x64.rpm 14 vi /etc/profile 15 source /etc/profile 16 java -versio 17 java -version 18 ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa 19 cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys 20 cd /root/.ssh 21 scp id_dsa.pub cMaster:/tmp/ 22 cp /mnt/hgfs/setup/hosts2.txt /etc/hosts 23 scp id_dsa.pub cMaster:/tmp/ 24 cat /tmp/id_dsa.pub >>authorized_keys 25 service iptables stop 26 chkconfig iptables off 27 chkconfig --list iptables 28 service iptables status 29 setenforce 0 30 vi /etc/selinux/config 31 yum install ntp -y 32 chkconfig ntpd on 33 service ntpd start 34 ntpdate -u s2m.time.edu.cn 35 yum install chkconfig python bind-utils psmisc libxslt zlib sqlite cyrus-sasl-plain cyrus-sasl-gssapi fuse fuse-libs redhat-lsb -y 36 mkdir -p /opt/cloudera-manager 37 tar -zxvf /opt/cloudera-manager-el6-cm5.9.3_x86_64.tar.gz -C /opt/cloudera-manager/ 38 useradd --system --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm 39 cd /opt/cloudera-manager/cm-5.12.1/etc/cloudera-scm-agent/ 40 cd /opt/cloudera-manager/cm-5.9.3/etc/cloudera-scm-agent/ 41 vi config.ini 42 mkdir -p /opt/cloudera/parcels 43 chown cloudera-scm:cloudera-scm /opt/cloudera/parcels 44 echo never > /sys/kernel/mm/transparent_hugepage/enabled 45 echo never > /sys/kernel/mm/transparent_hugepage/defrag 46 vi /etc/rc.local 47 echo 0 > /proc/sys/vm/swappiness 48 cd /opt/cloudera-manager/cm-5.12.1/etc/init.d/ 49 ifconfig 50 cd /opt/cloudera-manager/cm-5.9.3/etc/init.d/ 51 ./cloudera-scm-agent start 52 history





你可能感兴趣的:(CDH 5.9.3 集群配置)