大数据学习之路--搭建hadoop集群一键免密登录，以及分发部署脚本

关于hadoop集群搭建的内容比较多，这里主要介绍三个与集群搭建相关的三个shell脚本。

1. 一键生成免密登录脚本

① vi  sshpass.sh
#!/bin/bash
# Name     : sshpass.sh
# Time     : 17/03/2018
# Author   : zsd
# Purpose  : For fast and easy setup of the SSH Passwordless access among all the nodes
#            in a cluster. 
# User     : Any user you are performing the test! Better to settup a separate user from your
#            working env to avoid troubles!!! "hadoop" is used in this example, and you can change it
#            via the export virable "USER=hadoop"
# Attention: The test env is assuming that each $USER on each $HOST is usring the same password!
#            And this likely makes sense as no body want to put more trouble on this.
# Usage    : 1st, make sure the script has the execute permisison "chmod +x ssh_pass.sh"
#            ./ssh_pass.sh password
#          : 2nd, ensure the "ssh4slaves.sh" script is with ssh_pass.sh for all nodes setup!!!
#          : 3rd, "expect" has to be installed on all the nodes for the SSH config

export FILELOC="/hadoop"
export SLAVESFILE="$FILELOC/sshslaves"
export HOSTS=`cat $FILELOC/sshhosts`
export SLAVES=`cat $FILELOC/sshslaves`
export SSH4SLAVESCRIPT="$FILELOC/ssh4slaves.sh"
export MASTER=node1
export USER=hadoop
export PASSWD=$1
export SSHLOC="$FILELOC/.ssh/"
export RSAFILE="$FILELOC/.ssh/id_rsa"
export RSAPUBFILE="$FILELOC/.ssh/id_rsa.pub"
export AUTHFILE="$FILELOC/.ssh/authorized_keys"
export EXPECTCHK=`rpm -qa expect | wc -l`

#
if [ $EXPECTCHK != 1 ]
  then
  echo ''
  echo "########################################################################################"
  echo "Please install the \"expect\" package first on all nodes to allow the script to run!!!"
  echo "yum -y install expect"
  echo "########################################################################################"
else
  if [ -e $RSAFILE ]
    then
    echo "########################################################################################"
    echo "Attention: This is for TEST ONLY, please fully test it before applying it to PROD"
    echo "environment!!! OR you might get in trouble!!!"
    echo ''
    echo "BETTER TO HAVE A NEW USER FOR THE TEST TO AVOID DESTROYING YOUR ENVIRONMENT!"
    echo ''
    echo "Please manually delete the ssh related file on each host before executing the script!!!"
    echo ''
    for host in $HOSTS
    do 
    echo "Please run command on $host: rm -rf $SSHLOC"
    done
    echo "########################################################################################"
  else
  # Just generate 
    for host in $HOSTS
    do
      if [ $host = "$MASTER" ]
        then 
        echo ''
        echo "###########################################################"
        echo "Generating RSA keys for MASTER host $MASTER"
        echo "###########################################################"
        echo ''
        expect -c "
            set timeout 1
            spawn ssh $USER@$host
            expect \"yes/no\"
            send -- \"yes\r\"
            expect \"password:\"
            send -- \"$PASSWD\r\"
            expect \"#\"
            send \"ssh-keygen -t rsa -P '' -f $RSAFILE\r\"
            expect \"#\"
            send \"ssh-copy-id -i $RSAPUBFILE $MASTER\r\"
            expect \"password:\"
            send -- \"$PASSWD\r\"
            expect eof
         "
        else
         echo ''
         echo "###########################################################"
         echo "Generating RSA keys for all OTHER hosts..."
         echo "hostname is $host"
         echo "###########################################################"
         echo ''
         expect -c "
           set timeout 1
           spawn ssh $USER@$host
           expect \"yes/no\"
           send -- \"yes\r\"
           expect \"password:\"
           send -- \"$PASSWD\r\"
           expect \"#\"
           send \"ssh-keygen -t rsa -P '' -f $RSAFILE\r\"
           expect \"#\"
           send \"ssh-copy-id -i $RSAPUBFILE $MASTER\r\"
           expect \"yes/no\"
           send -- \"yes\r\"
           expect \"password:\"
           send -- \"$PASSWD\r\"
           expect eof
           "
         fi
    done            

    ### 
    for host in $SLAVES 
    do
        echo ''
        echo "############################################################################"
        echo "Copying authorized_keys to host $host from the MASTER host $MASTER..."
        echo "############################################################################"
        echo ''
        expect -c "
        set timeout 3
        spawn scp $AUTHFILE "$USER@$host:$SSHLOC"
        expect \"password:\"
        send -- $PASSWD\r
        expect eof
        "
    done

  #
    for host in $SLAVES
    do
      echo ''
      echo "############################################################################"
      echo "Distributing the $SLAVESFILE file to slave host $host..."
      echo "############################################################################"
      echo ''
      scp $SLAVESFILE "$host:$FILELOC"
      echo ''
      echo "############################################################################"
      echo "Distributing the $SSH4SLAVESCRIPT script to slave host $host..."
      echo "############################################################################"
      echo ''
      scp $SSH4SLAVESCRIPT "$host:$FILELOC"
    done


    for host in $SLAVES
    do
      echo ''
      echo "############################################################################"
      echo "Working on the slaves node $host to ensure no prompt for the "yes/no" question..."
      echo "############################################################################"
      echo ''
      ssh -q $USER@$host $SSH4SLAVESCRIPT
    done

    ### Check whether the Passwordless ssh works ###
    for host in $HOSTS
    do
      echo ''
      echo "############################################################################"
      echo "Check whether the Passwordless SSH works for $host..."
      echo "############################################################################"
      echo ''
      ssh $host uname -a && date
    done
  fi
fi

② vi ssh4slaves.sh
#!/bin/bash
# Name     : ssh4slaves.sh
# Time     : 17/03/2018
# Author   : zsd
# Purpose  : For fast and easy setup of the SSH Passwordless access among all the slave nodes
#            in a cluster. Mainly to ensure no prompt for "yes/no" again!!!
# User     : Any user you are performing the test! Better to settup a separate user from your
#            working env to avoid troubles!!! "hadoop" is used in this example, and you can change it
#            via the export virable "USER=hadoop"
# Attention: The test env is assuming that each $USER on each $HOST is usring the same password!
#            And this likely makes sense as no body want to put more trouble on this.
# Usage    : This script is called by the main script "ssh_pass.sh"
#            1st, make sure the script has the execute permisison "chmod +x ssh4slaves.sh" before
#            distributing it to other slaves node.
#            2nd, Remember to change variable "PASSWORD" before start the main script "sshpass.sh" 

export FILELOC="/hadoop"
export SLAVES=`cat $FILELOC/sshslaves`
export USER=hadoop
export PASSWD=zsd

for host in $SLAVES
do
    echo ''
    echo "Ensure ssh passwordless works among all slave nodes..."
    echo ''
    expect -c "
        set timeout 1
        spawn ssh $USER@$host
        expect \"yes/no\"
        send -- \"yes\r\"
        expect eof
     "
done

sshhosts主机名清单

 ③ vi sshhosts
node1
node2
node3
node4
node5
node6

slave清单，（个数比主机名清单少（master）1个）

vi sshslaves
node2
node3
node4
node5
node6

使用方法：

1. 把这4个文件放在家目录下（如果是root用户免密登录，就放在/root/ 下，我是hadoop用户免密登录家目录为 /home/hadoo/）
2. sshslaves要放在所有的节点的对应节点的家目录如 /home/hadoo/
3. sudo chown u+x *.sh赋予可执行权限
4. 如果是window环境编辑的shell脚本，请在linux环境下执行一次 sed -i ‘s/\r$//’ *.sh
   -路径 用户名 密码 sshhosts sshslaves根据自己的改正确。
5. 其他要求，用户名需要相同
6. 如果你的集群传输不稳定建议将所有 timeout 改大写15~30，这个代表在延迟时间内，如果没有捕获到期望值，直接执行sed命令。
7. 最后执行 ./sshpass.sh zsd (zsd 改成你自己的用户密码)

2 . 命令、文件和目录分发脚本
命令分发脚本：

① vi runRemoteCmd.sh
#!/bin/bash
#set -x

if [ $# -lt 2 ]
then 
  echo "Usage: ./runRemoteCmd.sh Command MachineTag"
  echo "Usage: ./runRemoteCmd.sh Command MachineTag confFile"
  exit 
fi

cmd=$1
tag=$2
if [ 'a'$3'a' == 'aa' ]
then

  confFile=/home/hadoop/tools/deploy.conf
else 
  confFile=$3
fi

if [ -f $confFile ]
then
    for server in `cat $confFile|grep -v '^#'|grep ','$tag','|awk -F',' '{print $1}'` 
    do
       echo "*******************$server***************************"
       ssh $server "source /etc/profile; $cmd"
    done 
else
  echo "Error: Please assign config file or run deploy.sh command with deploy.conf in same directory"
fi

文件和目录分发脚本：

② vi deploy.sh
#!/bin/bash
#set -x

if [ $# -lt 3 ]
then 
  echo "Usage: ./deply.sh srcFile(or Dir) descFile(or Dir) MachineTag"
  echo "Usage: ./deply.sh srcFile(or Dir) descFile(or Dir) MachineTag confFile"
  exit 
fi

src=$1
dest=$2
tag=$3
if [ 'a'$4'a' == 'aa' ]
then
  confFile=/home/hadoop/tools/deploy.conf
else 
  confFile=$4
fi

if [ -f $confFile ]
then
  if [ -f $src ]
  then
    for server in `cat $confFile|grep -v '^#'|grep ','$tag','|awk -F',' '{print $1}'` 
    do
       scp $src $server":"${dest}
    done 
  elif [ -d $src ]
  then
    for server in `cat $confFile|grep -v '^#'|grep ','$tag','|awk -F',' '{print $1}'` 
    do
       scp -r $src $server":"${dest}
    done 
  else
      echo "Error: No source file exist"
  fi

else
  echo "Error: Please assign config file or run deploy.sh command with deploy.conf in same directory"
fi

③vi deploy.conf 第一列为主机名列表，后面是别名，后面的逗号不能丢
#### NOTES
# There is crontab job using this config file which would compact log files and remove old log file.
# please be  carefully while modifying this file until you know what crontab exactly do
#hdp
node1,jdk,hadoop,habse,es,rm,1,nn,
node2,jdk,hadoop,hbase,es,rm,2,nn,
node3,jdk,hadoop,hbase,zk,es,3,dn,spark,
node4,jdk,hadoop,hbase,zk,4,dn,spark,
node5,jdk,hadoop,hbase,zk,5,dn,hive,spark,
node6,jdk,hadoop,flume,kafka,hive,sqoop,6,dn,spark,

使用说明：

• 两个脚本需要放在/home/hadoop/tools/目录下，另外建议为tools配置环境变量，这样可以在任意目录下执行
• 比如要将zookeeper安装包发送到所有节点，/home/hadoop/app/zookeeper
  deploy.sh zookeeper /home/hadoop/app/ zk
• 启动zookeeper： runRemoteCmd.sh “zkServer start” zk