zookeeper+kafka集群一键安装脚本

1. zookeeper_kafka_install.sh

#!/bin/bash

read -p "is need to deploy cluster environment[yes|no]: " booleanResult
if [ $booleanResult = "yes" ] || [ $booleanResult = "no" ];then
	echo "input is: "$booleanResult
else
	echo "only input [yes|no]"
	exit
fi
## root path
INSTALL_BESE_URL=/home/bigdata/app
## deployment template path
TGZ_PACKEGER_URL=/home/bigdata/app/base/dev
## install path
INSTALL_PACKEGER_URL=/home/bigdata/app/base/install
## zookeeper tgz name
ZOOKEEPER_TGZ=Auto.BigData.SystemBase.zookeeper.tgz
## kafka tgz name
KAFKA_TGZ=Auto.BigData.SystemBase.kafka.tgz
## zookeeper name
ZOOKEEPER_NAME=zookeeper-3.4.14
## kafka name
KAFKA_NAME=kafka_2.11-1.1.1
## kafka config file path
KAFKA_CONFIG_FILE_PATH=$TGZ_PACKEGER_URL/$KAFKA_NAME/config/server.properties
## remove old file
rm -rf /home/bigdata/app/base
PID=$(ps aux | grep kafka | grep -v grep | awk '{print $2}' )
echo "kafka runnig id is "$PID
## get local ip 
LOCAL_IP=$(ifconfig | grep  inet | grep netmask | grep broadcast | awk '{print $2}' | sed -n '1p')
echo "local ip is "$LOCAL_IP	
LOCAL_PATH=$(pwd)
echo "local path pwd:"$LOCAL_PATH
if [ ! -d $TGZ_PACKEGER_URL ];then
	echo "$TGZ_PACKEGER_URL is not create"
	mkdir -p $TGZ_PACKEGER_URL
else
	echo "$TGZ_PACKEGER_URL is exist"
fi
if [ -e $LOCAL_PATH/$ZOOKEEPER_TGZ ];then
	tar -zxvf $LOCAL_PATH/$ZOOKEEPER_TGZ -C $TGZ_PACKEGER_URL
else 
	echo "zookeeper tgz is not exist,path=$INSTALL_BESE_URL/$ZOOKEEPER_TGZ"
	exit
fi

if [ -e $LOCAL_PATH/$KAFKA_TGZ ];then
	tar -zxvf $LOCAL_PATH/$KAFKA_TGZ -C $TGZ_PACKEGER_URL
else 
	echo "kafka tgz is not exist,path=$INSTALL_BESE_URL/$KAFKA_TGZ"
	exit
fi

if [ ! -d $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data ];then
	mkdir $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data
fi
START_ZOOKEEPER_FILE_NAME=zookeeper_start.sh
START_KAFKA_FILE_NAME=kafka_start.sh
KILL_ZK_KF_PID_FILE_NAME=kill_zk_kf_pid.sh
cp $LOCAL_PATH/$START_ZOOKEEPER_FILE_NAME $TGZ_PACKEGER_URL
cp $LOCAL_PATH/$START_KAFKA_FILE_NAME $TGZ_PACKEGER_URL
cp $LOCAL_PATH/$KILL_ZK_KF_PID_FILE_NAME $TGZ_PACKEGER_URL
chmod -R 777 $TGZ_PACKEGER_URL/*
echo "end tar tgz!!!!!"


## check user input ip
input_ip_list=""
function check_input_ip_info {
	echo "========start vail ip is it repeated========"
	for ip in $input_ip_list;do
		echo "-------for ip="$ip",inputIpInfo="$inputIpInfo"-----------"
		if [ $ip = $inputIpInfo ]; then
			echo "input ip is already exist: "$input_ip_list
			return 1
		fi
	done
	echo "========end vail ip is it repeated========"
	ping -c 1  -w 100 $inputIpInfo
	if [ $? -eq 0 ];then
		echo "input ip is right"
		input_ip_list=$input_ip_list" $inputIpInfo"
	else
		echo "input ip is error,please reinput"     
	fi
}
## loop set zookeeper property
total_num=0
function loop_set_zookeeper_server_config {
	echo "start set zookeeeper config"
	for ip_obj in $input_ip_list;
	do
		echo $ip_obj
		let total_num++
		server_ip="server.$total_num=$ip_obj:2888:3888"
		echo $server_ip >> $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/conf/zoo.cfg
		echo "zookeeper server ip $total_num=$server_ip"
	done
	echo "end set zookeeeper config"
}


## set kafka server.properties property
function loop_set_kafka_config {
	echo "start set kafka config"
	echo 'num.partitions='$total_num >> $KAFKA_CONFIG_FILE_PATH
	echo 'offsets.topic.replication.factor='$total_num >> $KAFKA_CONFIG_FILE_PATH
	echo 'transaction.state.log.replication.factor='$total_num >> $KAFKA_CONFIG_FILE_PATH
	echo 'transaction.state.log.min.isr='$total_num >> $KAFKA_CONFIG_FILE_PATH
	echo 'default.replication.factor='$total_num >> $KAFKA_CONFIG_FILE_PATH

	echo 'num.network.threads=3' >> $KAFKA_CONFIG_FILE_PATH
	echo 'num.io.threads=8' >> $KAFKA_CONFIG_FILE_PATH
	echo 'socket.send.buffer.bytes=102400' >> $KAFKA_CONFIG_FILE_PATH
	echo 'socket.receive.buffer.bytes=102400' >> $KAFKA_CONFIG_FILE_PATH
	echo 'socket.request.max.bytes=104857600' >> $KAFKA_CONFIG_FILE_PATH
	echo 'log.dirs='$INSTALL_PACKEGER_URL'/'$KAFKA_NAME'/logs' >> $KAFKA_CONFIG_FILE_PATH
	echo 'num.recovery.threads.per.data.dir=1' >> $KAFKA_CONFIG_FILE_PATH
	echo 'log.retention.hours=48' >> $KAFKA_CONFIG_FILE_PATH
	echo 'log.retention.bytes=214748364800' >> $KAFKA_CONFIG_FILE_PATH
	echo 'log.segment.bytes=1073741824' >> $KAFKA_CONFIG_FILE_PATH
	echo 'log.retention.check.interval.ms=300000' >> $KAFKA_CONFIG_FILE_PATH
	echo 'zookeeper.connection.timeout.ms=6000' >> $KAFKA_CONFIG_FILE_PATH
	echo 'group.initial.rebalance.delay.ms=0' >> $KAFKA_CONFIG_FILE_PATH
	zookeeepr_ip=""
	zookeeper_flag_num=0
	for ip_info in $input_ip_list;
	do
		let zookeeper_flag_num++
		if [ $zookeeper_flag_num -eq 1 ];then
			zookeeepr_ip=$ip_info":2181"
		else
			zookeeepr_ip=$zookeeepr_ip",$ip_info:2181"
		fi
	done
	echo 'zookeeper.connect='$zookeeepr_ip >> $KAFKA_CONFIG_FILE_PATH
	echo "end set kafka config"
}

## local install funtion
function generate_local_install {
	if [ -d $INSTALL_PACKEGER_URL ];then
	    rm -rf $INSTALL_PACKEGER_URL/*
	else
		mkdir -p $INSTALL_PACKEGER_URL
	fi
	cp $TGZ_PACKEGER_URL/* $INSTALL_PACKEGER_URL/
	$INSTALL_PACKEGER_URL/$KILL_ZK_KF_PID_FILE_NAME
}
## remote install function
function generate_remote_install {
	echo "#####################check install package path is exist ip:"$ip_adr"############################"
	ssh bigdata@$ip_adr "[ -d "$INSTALL_PACKEGER_URL" ] && rm -rf "$INSTALL_PACKEGER_URL"/* || mkdir -p" $INSTALL_PACKEGER_URL
	echo "#####################send remote install package ip:"$ip_adr"############################"
	scp -r $TGZ_PACKEGER_URL/* bigdata@$ip_adr:$INSTALL_PACKEGER_URL
	echo "#####################kill old kafka zookeeper process id ip:"$ip_adr"############################"
	ssh bigdata@$ip_adr $INSTALL_PACKEGER_URL/$KILL_ZK_KF_PID_FILE_NAME
}
## start zookeeper function
function start_zookeeper_fun {
	for ip_adr_zookeeper in $input_ip_list;do
		if [ $ip_adr_zookeeper = $LOCAL_IP ];then
			echo "install kafka is local ip:"$ip_adr_zookeeper
			$INSTALL_PACKEGER_URL/$START_ZOOKEEPER_FILE_NAME
		else
			echo "install zookeeper is remote ip:"$ip_adr_zookeeper
			ssh bigdata@$ip_adr_zookeeper $INSTALL_PACKEGER_URL/$START_ZOOKEEPER_FILE_NAME
		fi
	done
}
## start kafka function
function start_kafka_fun {
	for ip_adr_kafka in $input_ip_list;do
		if [ $ip_adr_kafka = $LOCAL_IP ];then
			echo "install kafka is local ip:"$ip_adr_kafka
			$INSTALL_PACKEGER_URL/$START_KAFKA_FILE_NAME
		else
			echo "install kafka is remote ip:"$ip_adr_kafka
			ssh bigdata@$ip_adr_kafka $INSTALL_PACKEGER_URL/$START_KAFKA_FILE_NAME
		fi
	done
}
## loop send zookee+kafka install packages to remote servers
send_flag_num=0
function loop_send_kafka_zookeeper {
	echo "====================start send remote zookeeper========================"
	echo "ips="$input_ip_list
	for ip_adr in $input_ip_list;
	do
		echo "delete kafka server.properties and zookeeper myid file"
		rm -rf $KAFKA_CONFIG_FILE_PATH
		rm -rf $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data/myid
		echo "recreate kafka server.properties and zookeeper myid file"
		touch $KAFKA_CONFIG_FILE_PATH
		touch $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data/myid
		echo "set kafka attr"
		echo 'broker.id='$send_flag_num >> $KAFKA_CONFIG_FILE_PATH
		echo 'port=9092' >> $KAFKA_CONFIG_FILE_PATH
		echo 'host.name='$ip_adr >> $KAFKA_CONFIG_FILE_PATH
		loop_set_kafka_config
		
		let send_flag_num++
		echo $send_flag_num >> $TGZ_PACKEGER_URL/$ZOOKEEPER_NAME/data/myid
		
		if [ $ip_adr = $LOCAL_IP ];then
			echo "start local install ip: $ip_adr"
			generate_local_install
			echo "end local install ip: $ip_adr"
		else
			echo "start remote install ip: $ip_adr"
			generate_remote_install
			echo "end remote install ip: $ip_adr"
		fi
		echo "send ip: $ip_adr"
	done
	echo "====================end send remote ips=$input_ip_list,start execute path /home/bigdata/app/base/install/zookeeper_kafka_start.sh========================"
	echo "======================start install zookeeepr=========================="
	start_zookeeper_fun
	echo "======================end install zookeeepr=========================="
	
	echo "======================start install kafka=========================="
	start_kafka_fun
	echo "======================end install kafka=========================="
}


if [ $booleanResult = "yes" ];then
## need to deploy cluster version
	echo "start colony set"
	while true
	do
        read -p "Please enter the cluster IP address,exit input[exit]: " inputIpInfo
		if [ $inputIpInfo = "exit" ];then
			echo "break ip input loop"	
		    break
		else
			check_input_ip_info
		fi 
	done
	echo "user input finish"
	if [ -z $input_ip_list ];then
		echo "ips is empty"
		exit
	fi
	loop_set_zookeeper_server_config
	loop_send_kafka_zookeeper
	echo "end colony ips is: "$input_ip_list
else
## simply deploy the stand-alone version
	echo "start not colony set"
	inputIpInfo=$LOCAL_IP
	echo 'broker.id='$send_flag_num >> $KAFKA_CONFIG_FILE_PATH
	echo 'port=9092' >> $KAFKA_CONFIG_FILE_PATH
	echo 'host.name='$inputIpInfo >> $KAFKA_CONFIG_FILE_PATH
	total_num=1
	loop_set_kafka_config
	if [ ! -d $INSTALL_PACKEGER_URL ];then
		mkdir -p $INSTALL_PACKEGER_URL
	fi
	rm -rf $INSTALL_PACKEGER_URL/*
	mv $TGZ_PACKEGER_URL/* $INSTALL_PACKEGER_URL/
##  execute start cmd
	$INSTALL_PACKEGER_URL/$KILL_ZK_KF_PID_FILE_NAME
	$INSTALL_PACKEGER_URL/$START_ZOOKEEPER_FILE_NAME
	$INSTALL_PACKEGER_URL/$START_KAFKA_FILE_NAME
    echo "end not colony set"
fi
rm -rf $TGZ_PACKEGER_URL

2. kill_zk_kf_pid.sh

#!/bin/bash

## kafka process id
KAFKA_PID=$( ps aux | grep kafka_2.11-1.1.1 | grep -v grep | awk '{print $2}' )
## zookeeper process id
ZOOKEEPER_PID=$( ps aux | grep zookeeper-3.4.14 | grep -v grep | awk '{print $2}' )

## kill old zookeeper process id
for zk_pid in $ZOOKEEPER_PID;
do
	echo "kill process zookeeper pid:"$zk_pid
	kill -9 $zk_pid
done
## kill old kafka process id
for ka_pid in $KAFKA_PID;
do
	echo "kill process kafka pid:"$ka_pid
	kill -9 $ka_pid
done

3. zookeeper_start.sh

#!/bin/bash

## install path
INSTALL_PACKEGER_URL=/home/bigdata/app/base/install
## zookeeper name
ZOOKEEPER_NAME=zookeeper-3.4.14
## kafka name
KAFKA_NAME=kafka_2.11-1.1.1
## kafka process id
KAFKA_PID=$( ps aux | grep kafka_2.11-1.1.1 | grep -v grep | awk '{print $2}' )
## zookeeper process id
ZOOKEEPER_PID=$( ps aux | grep zookeeper-3.4.14 | grep -v grep | awk '{print $2}' )
## zookeeper service path
ZKFILE="/usr/lib/systemd/system/zookeeper.service"
echo "delete zookeeper service file"
sudo rm -rf $ZKFILE
## determine whether or not JDK is installed
which java > /dev/null 2>&1
if [ $? -eq 0 ];then
    echo "java was installed"
else
    echo "java isn't install,please install java first"
    exit
fi

## zookeeper environment variable path
ZKBIN_PATH="export PATH=${PATH}:/home/bigdata/app/base/install/"$ZOOKEEPER_NAME"/bin"
## judge zookeeper environment variable is exist
sudo grep -i "/home/bigdata/app/base/install/"$ZOOKEEPER_NAME"/bin" /etc/profile
## set zookeeper environment variable
if [ $? -eq 0 ];then
    echo $ZKBIN_PATH" is exist,skip"
else
    sudo chmod 745 /etc/profile
    sudo chmod 777 /etc/profile
    sudo echo $ZKBIN_PATH >> /etc/profile
	source /etc/profile
fi
echo "chmod kafka and zookeeper bin file power"
chmod -R 777 /home/bigdata/app/base/install/kafka_2.11-1.1.1/bin/*
chmod -R 777 /home/bigdata/app/base/install/kafka_2.11-1.1.1/config/*
chmod -R 777 /home/bigdata/app/base/install/kafka_2.11-1.1.1/libs/*
chmod -R 777 /home/bigdata/app/base/install/kafka_2.11-1.1.1/*
chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/bin/*
chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/data/*
chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/lib/*
chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/conf/*
chmod -R 777 /home/bigdata/app/base/install/zookeeper-3.4.14/*

## set up zookeeper boot start
function set_zookeeper_system {
	echo "start set zookeeper service config"
	sudo sh -c 'echo "[Unit]" > '$ZKFILE
	sudo sh -c 'echo "Description=zookeeper.service" >> '$ZKFILE
	sudo sh -c 'echo "After=network.target" >> '$ZKFILE
	sudo sh -c 'echo "[Service]" >> '$ZKFILE
	sudo sh -c 'echo "User=bigdata" >> '$ZKFILE
	sudo sh -c 'echo "Type=forking" >> '$ZKFILE
	sudo sh -c 'echo "Environment=ZOO_LOG_DIR=/home/bigdata/app/base/install/zookeeper-3.4.14/log" >> '$ZKFILE
	sudo sh -c 'echo "ExecStart=/home/bigdata/app/base/install/zookeeper-3.4.14/bin/zkServer.sh start" >> '$ZKFILE
	sudo sh -c 'echo "ExecStop=/home/bigdata/app/base/install/zookeeper-3.4.14/bin/zkServer.sh stop" >> '$ZKFILE
	sudo sh -c 'echo "ExecReload=/home/bigdata/app/base/install/zookeeper-3.4.14/bin/zkServer.sh restart" >> '$ZKFILE
	sudo sh -c 'echo "[Install]" >> '$ZKFILE
	sudo sh -c 'echo "WantedBy=multi-user.target" >> '$ZKFILE

	sudo systemctl daemon-reload
	echo "end set zookeeper service config"	
	echo "==========start zookeeper process=========="
	sudo systemctl start zookeeper
	if [ $? -eq 0 ];then
		echo "zookeeper was installed success"
	else
		echo "zookeeper was installed false,please check your install file"
	fi
}

sudo touch $ZKFILE
set_zookeeper_system

4. kafka_start.sh

#!/bin/bash

KAFKAFILE="/usr/lib/systemd/system/kafka.service"

echo "delete kafka service file"
sudo rm -rf $KAFKAFILE

## set up kafka boot start
function set_kafka_system {
	echo "start set kafka service config"
	sudo sh -c 'echo "[Unit]"  > '$KAFKAFILE
	sudo sh -c 'echo "Description=kafka.service"  >> '$KAFKAFILE
	sudo sh -c 'echo "[Service]"  >> '$KAFKAFILE
	sudo sh -c 'echo "Type=forking" >> '$KAFKAFILE
	sudo sh -c 'echo "User=bigdata"  >> '$KAFKAFILE
	sudo sh -c 'echo "Environment=JMX_PORT=9988" >> '$KAFKAFILE
	sudo sh -c 'echo "ExecStart=/home/bigdata/app/base/install/kafka_2.11-1.1.1/bin/kafka-server-start.sh -daemon /home/bigdata/app/base/install/kafka_2.11-1.1.1/config/server.properties"  >> '$KAFKAFILE
	sudo sh -c 'echo "[Install]"  >> '$KAFKAFILE
	sudo sh -c 'echo "WantedBy=multi-user.target"  >> '$KAFKAFILE
	sudo systemctl daemon-reload
	echo "end set kafka service config"
	echo "==========start kafka process=========="
	sudo systemctl start kafka
}
sudo touch $KAFKAFILE
set_kafka_system