kafka自动化脚本部署

一、背景

    如果不采用CDH或者ambari等平台来部署kafka的话，一台一台的安装，一台一台的改配置，真的是一个非常让人头疼的事情呢，经领导提示，为什么不写个shell脚本来进行一件部署呢

二、java自动部署

if [ ! -d /usr/java/jdk1.8.0_121 ];then
	mkdir -p  /root/software/
	wget -q http://192.168.10.152/hadoop/jdk-8u121-linux-x64.tar.gz -O /root/software/jdk-8u121-linux-x64.gz
	if [ $? -eq 0 ];then
		echo "软件下载成功"
		mkdir -p /usr/java/
		tar zxf /root/software/jdk-8u121-linux-x64.gz -C /usr/java/
		echo  "export JAVA_HOME=/usr/java/jdk1.8.0_121"  >> /etc/profile
		echo 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile
		source /etc/profile
		echo "JAVA install is ok!!"
	else
		echo "软件下载失败"
		exit 1	
	fi
fi
source /etc/profile

三、Zookeeper自动部署（同下面kafka的自动部署一样，需要将各机器共同的配置放在一个公共的配置文件里，然后进行操作）

if [ $# == 0 ];then
	echo "please pointed config location!"
	exit 1;
fi
confPath=$1

ips=`hostname -i`
for ip in ${ips[@]}
do
        for line in $(cat $confPath)
        do
                result=$(echo $line | grep "${ip}")
                if [ "$result" != "" ]
                then
                        myid=$(echo $line|cut -d = -f 1 |cut -d . -f 2)
                fi
        done
done

if [ !$myid ];then
	echo "please check zoo.cfg!"
	exit 1;
fi
#################################zookeeper 部署及相关python包部署##################################

wget http://192.168.10.152/hadoop/zookeeper-3.4.8.tar.gz -O /usr/local/zookeeper-3.4.8.tar.gz
tar -xzf /usr/local/zookeeper-3.4.8.tar.gz -C /usr/local
ln -s /usr/local/zookeeper-3.4.8 /usr/local/zookeeper
rm -rf /usr/local/zookeeper-3.4.8.tar.gz


grep "export ZOOKEEPER_HOME=/usr/local/zookeeper" /etc/profile > /dev/null
if [ $? -eq 1 ]; then
    echo "export ZOOKEEPER_HOME=/usr/local/zookeeper" >> /etc/profile
fi

grep "export PATH=\\$\ZOOKEEPER_HOME/bin:\\$\PATH" /etc/profile > /dev/null
if [ $? -eq 1 ]; then
    echo "export PATH=\$ZOOKEEPER_HOME/bin:\$PATH" >> /etc/profile
fi

source /etc/profile

cp $ZOOKEEPER_HOME/bin/zkEnv.sh $ZOOKEEPER_HOME/bin/zkEnv.sh.bak
sed -i 's\ZOO_LOG_DIR="."\ZOO_LOG_DIR="$ZOOBINDIR/../logs"\g' $ZOOKEEPER_HOME/bin/zkEnv.sh

cp $ZOOKEEPER_HOME/conf/log4j.properties $ZOOKEEPER_HOME/conf/log4j.properties.bak
sed -i 's/zookeeper.root.logger=INFO, CONSOLE/zookeeper.root.logger=INFO, ROLLINGFILE/g' $ZOOKEEPER_HOME/conf/log4j.properties


################init config ##############
# cat <$ZOOKEEPER_HOME/conf/zoo.cfg
# tickTime=2000
# the directory where the snapshot is stored
# dataDir=/data/zookeeper/data
# the directory where the transaction log is stored
# aLogDir=/data/zookeeper/log
# entPort=6181
# tLimit=10
# cLimit=5
# ClientCnxns=0
cat $confPath>$ZOOKEEPER_HOME/conf/zoo.cfg
cat $ZOOKEEPER_HOME/conf/zoo.cfg

################init manage script ##############

mkdir -p /root/scripts
mkdir -p /data/zookeeper/data
touch /root/scripts/zookeeper
cat < /root/scripts/zookeeper
#/bin/bash   

ZOOKEEPER_HOME=/usr/local/zookeeper

case \$1 in  
          start) \$ZOOKEEPER_HOME/bin/zkServer.sh start;;  
          stop) \$ZOOKEEPER_HOME/bin/zkServer.sh stop;;                                 
          status) \$ZOOKEEPER_HOME/bin/zkServer.sh status;;  
          restart) \$ZOOKEEPER_HOME/bin/zkServer.sh stop&\$ZOOKEEPER_HOME/bin/zkServer.sh stop;;  
              *)  echo "require start|stop|status|restart";;  
esac
EOF
chmod +x /root/scripts/zookeeper

/root/scripts/zookeeper

grep "nohup /root/scripts/zookeeper start &" /etc/rc.local > /dev/null
if [ $? -eq 1 ]; then
    echo 'nohup /root/scripts/zookeeper start &' >> /etc/rc.local
fi


echo "ZOOKEEPR install ok!"

echo $myid > /data/zookeeper/data/myid

四、kafka自动部署脚本

#################################zookeeper 部署及相关python包部署##################################
###########判断zookeeper是否安装############
#zookeeperPath="/user/local/zookeeper"
#if [ ! -d "$zookeeperPath" ]; then
#       source ./zookeeper.sh 
#fi
if [ $# == 0 ];then
        echo "please pointed config location!"
    exit 1;
fi

wget http://192.168.10.152/hadoop/kafka_2.11-0.11.0.1.tgz -O /usr/local/kafka_2.11-0.11.0.1.tgz
tar -xzf /usr/local/kafka_2.11-0.11.0.1.tgz -C /usr/local
ln -s /usr/local/kafka_2.11-0.11.0.1 /usr/local/kafka
rm -rf /usr/local/kafka_2.11-0.11.0.1.tgz


grep "export KAFKA_HOME=/usr/local/kafka" /etc/profile > /dev/null
if [ $? -eq 1 ]; then
    echo "export KAFKA_HOME=/usr/local/kafka" >> /etc/profile
fi

grep "export PATH=\\$\KAFKA_HOME/bin:\\$\PATH" /etc/profile > /dev/null
if [ $? -eq 1 ]; then
    echo "export PATH=\$KAFKA_HOME/bin:\$PATH" >> /etc/profile
fi


source /etc/profile > /dev/null

cp $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/bin/kafka-server-start.sh.bak
sed -i 's/"-Xmx1G -Xms1G"/"-Xmx12G -Xms12G"\n    export JMX_PORT="9999"/' $KAFKA_HOME/bin/kafka-server-start.sh

mv $KAFKA_HOME/config/server.properties $KAFKA_HOME/config/server.properties.bak
################init config ##############
broker_id=`hostname|cut -d - -f 5`
echo "broker.id=$broker_id">>$KAFKA_HOME/config/server.properties

ipaddr=`hostname -i`
echo "listeners=PLAINTEXT://$ipaddr:9092">>$KAFKA_HOME/config/server.properties

if [ $broker_id -le 123 ]; then echo "broker.rack=B0614">>$KAFKA_HOME/config/server.properties; fi
if [ $broker_id -gt 123 ] && [ $broker_id -le 126 ]; then echo "broker.rack=B0615">>$KAFKA_HOME/config/server.properties; fi

confPath=$1
cat $confPath>>$KAFKA_HOME/config/server.properties

################init manage script ##############

mkdir -p /root/scripts
# mkdir -p /data/kafka-logs
touch /root/scripts/Kafka
cat < /root/scripts/Kafka
#/bin/bash   

KAFKA_HOME=/usr/local/kafka

case \$1 in  
          start) \$KAFKA_HOME/bin/kafka-server-start.sh -daemon \$KAFKA_HOME/config/server.properties;;  
           stop) \$KAFKA_HOME/bin/kafka-server-stop.sh;;                                 
              *)  echo "require start|stop";;  
esac
EOF
chmod +x /root/scripts/Kafka
/root/scripts/Kafka

grep "nohup /root/scripts/Kafka start\ \&" /etc/rc.local > /dev/null
if [ $? -eq 1 ]; then
    echo 'nohup /root/scripts/Kafka start &' >> /etc/rc.local
fi


echo "KAFKA install ok!"

五、卸载Zookeeper、kafka脚本

##############kafka卸载###################
if [ $# == 0 ];then
        echo "please pointed config location!"
    exit 1;
fi

rm /usr/local/kafka

grep "export KAFKA_HOME=/usr/local/kafka" /etc/profile > /dev/null
if [ $? -eq 0 ]; then
    sed -i '/export KAFKA_HOME=\/usr\/local\/kafka/d' /etc/profile
fi

grep "export PATH=\\$\KAFKA_HOME/bin:\\$\PATH" /etc/profile > /dev/null
if [ $? -eq 0 ]; then
    sed -i '/export PATH=\$KAFKA_HOME\/bin:\$PATH/d' /etc/profile
fi

source /etc/profile > /dev/null

################init manage script ##############
logDir=`grep 'log.dirs' server.properties | awk -F'=' '{print $2}'`
if [ -d $logDir/ ]; then rm -r $logDir; fi

rm /root/scripts/Kafka

grep "nohup /root/scripts/Kafka start &" /etc/rc.local > /dev/null
if [ $? -eq 0 ]; then
    sed -i '/nohup\ \/root\/scripts\/Kafka\ start\ \&/d' /etc/rc.local
fi
rm -r /usr/local/kafka_2.11-0.11.0.1

echo "KAFKA unstall ok!"

################卸载Zookeeper ##############
if [ $# == 0 ];then
	echo "please pointed config location!"
    exit 1;
fi

confPath=$1
rm /usr/local/zookeeper

grep "export ZOOKEEPER_HOME=/usr/local/zookeeper" /etc/profile > /dev/null
if [ $? -eq 0 ]; then
    sed -i '/export ZOOKEEPER_HOME=\/usr\/local\/zookeeper/d' /etc/profile
fi

grep "export PATH=\\$\ZOOKEEPER_HOME/bin:\\$\PATH" /etc/profile > /dev/null
if [ $? -eq 0 ]; then
    sed -i '/export PATH=\$ZOOKEEPER_HOME\/bin:\$PATH/d'  /etc/profile
fi
source /etc/profile

dataDir=`grep "dataDir" $confPath|cut -d = -f2`
dataLogDir=`grep "dataLogDir" $confPath|cut -d = -f2`
rm -r $dataDir
rm -r $dataLogDir
rm /root/scripts/zookeeper

grep "nohup /root/scripts/zookeeper start &" /etc/rc.local > /dev/null
if [ $? -eq 0 ]; then
    sed -i '/nohup\ \/root\/scripts\/zookeeper\ start\ \&/d' /etc/rc.local
fi
rm -r /usr/local/zookeeper-3.4.8
echo "ZOOKEEPR unstall complete!"