linux基础环境及CDH配置

新公司没有大数据平台，申请了虚拟环境，部署基于cdh的大数据平台，只保存操作步骤，详细的前面帖子有；

1、做免密

ssh-keygen -t rsa

chmod 700 .ssh
cd .ssh
touch authorized_keys
chmod 644 authorized_keys

# copy多个机器的公钥
vi id_rsa.pub 

# 写入keys文件后分发
scp authorized_keys 10.191.247.46:/root/.ssh
scp authorized_keys 10.191.247.47:/root/.ssh
scp authorized_keys 10.191.247.48:/root/.ssh
scp authorized_keys 10.191.247.49:/root/.ssh
scp authorized_keys 10.191.247.50:/root/.ssh

2、修改dns(保证连接外网)

# Generated by NetworkManager
nameserver 211.138.24.66
nameserver 114.114.114.114
nameserver 8.8.8.8

scp /etc/resolv.conf 10.191.247.46:/etc
scp /etc/resolv.conf 10.191.247.47:/etc
scp /etc/resolv.conf 10.191.247.48:/etc
scp /etc/resolv.conf 10.191.247.49:/etc
scp /etc/resolv.conf 10.191.247.50:/etc

3、修改yum源--阿里源,安装ansible

# 如果yum源没法下载wget的rpm包：
http://www.rpmfind.net/linux/rpm2html/search.php?query=wget
rpm -ivh wgetxxxxxxxxxx

yum clean all
yum install -y wget
yum install -y lrzsz

mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo_bak
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum makecache	

4、修改主机名，ansible配置

# 主机名映射
vi /etc/hosts

10.191.247.45 hadoop01
10.191.247.46 hadoop02
10.191.247.47 hadoop03
10.191.247.48 hadoop04
10.191.247.49 hadoop05
10.191.247.50 hadoop06

# 主机名
vi /etc/hostname

# 分别永久变更
hostnamectl set-hostname hadoop01
hostnamectl set-hostname hadoop02
hostnamectl set-hostname hadoop03
hostnamectl set-hostname hadoop04
hostnamectl set-hostname hadoop05
hostnamectl set-hostname hadoop06

# 修改ansible配置
vi /etc/ansible/hosts

[hadoop]
10.191.247.45
10.191.247.46
10.191.247.47
10.191.247.48
10.191.247.49
10.191.247.50
# 验证
ansible hadoop -m command -a ' ls'

5、挂载data目录

ansible hadoop -m command -a 'mkfs.ext4 /dev/vdb'
ansible hadoop -m command -a 'mkdir /data'
ansible hadoop -m command -a 'mount /dev/vdb /data'

echo "/dev/vdb /data ext4 defaults  0  0 " >>/etc/fstab

6、ntp服务器

ansible hadoop -m command -a 'yum install -y ntp'
ansible hadoop -m command -a 'systemctl restart ntpd'
ansible hadoop -m command -a 'timedatectl set-timezone Asia/Shanghai '

#分发
scp /etc/ntp.conf 10.191.247.46:/etc
scp /etc/ntp.conf 10.191.247.47:/etc
scp /etc/ntp.conf 10.191.247.48:/etc
scp /etc/ntp.conf 10.191.247.49:/etc
scp /etc/ntp.conf 10.191.247.50:/etc

#测试
ansible hadoop -m command -a 'systemctl restart ntpd'
ansible hadoop -m command -a 'ntpq –p'
ansible hadoop -m command -a 'date'

7、防火墙

ansible hadoop -m command -a 'systemctl disable firewalld'

#vi /etc/selinux/config
SELINUX=disabled
SELINUXTYPE=targeted

8、opt文件夹（虚拟机opt文件夹存储不足，链接到data）

ansible hadoop -m command -a  'mkdir /opt/cloudera'
ansible hadoop -m command -a  'mv /opt/cloudera /opt/cloudera.bak'
ansible hadoop -m command -a  'mv /opt/cloudera.bak /data/cloudera'
ansible hadoop -m command -a  'ln -sv /data/cloudera /opt/cloudera'

9、http服务

ansible hadoop -m command -a 'yum -y install httpd'
ansible hadoop -m command -a 'systemctl start httpd '

ansible hadoop -m command -a 'mkdir /var/www/html/cm-6.1.0 '
ansible hadoop -m command -a 'mkdir /var/www/html/cdh-6.1.0'

ansible hadoop -m command -a ' yum -y install yum-utils createrepo'

createrepo /var/www/html/cm-6.1.0
chmod  -R 755 /var/www/html

10、安装cm

chmod +x cloudera-manager-installer.bin 
# 安装jdk

rpm -ivh oracle-j2sdk1.8-1.8.0+update141-1.x86_64.rpm 
rpm -ivh cloudera-manager-daemons-6.1.0-769885.el7.x86_64.rpm 
rpm -ivh cloudera-manager-server-6.1.0-769885.el7.x86_64.rpm 
yum install -y cloudera-manager-server-db-2-6.1.0-769885.el7.x86_64.rpm
rpm -ivh cloudera-manager-server-db-2-6.1.0-769885.el7.x86_64.rpm 
rpm -ivh cloudera-manager-agent-6.1.0-769885.el7.x86_64.rpm 
yum install -y perl             
yum install -y bind-utils             
yum install -y libxslt                
yum install -y cyrus-sasl-plain       
yum install -y cyrus-sasl-gssapi      
yum install -y fuse                   
yum install -y fuse-libs              
yum install -y /lib/lsb/init-functions
yum install -y mod_ssl                
yum install -y openssl-devel          
yum install -y python-psycopg2        
yum install -y MySQL-python is        

rm /etc/cloudera-scm-server/db.properties -rf

./cloudera-manager-installer.bin --skip_repo_package=1

tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
netstat -lnpt | grep 7180 
mv CDH-6.1.0-1.cdh6.1.0.p0.770702-el7.parcel.sha256 CDH-6.1.0-1.cdh6.1.0.p0.770702-el7.parcel.sha

sha1sum CDH-6.1.0-1.cdh6.1.0.p0.770702-el7.parcel | cut -d ' ' -f 1 > CDH-6.1.0-1.cdh6.1.0.p0.770702-el7.parcel.sha