使用ansible 搭建kafka集群

 软件版本：

ansible
zookeeper
kafka

 

下载代码：
$git clone https://github.com/pingcap/thirdparty-ops

安装之前的准备工作：
# yum -y install epel-release ansible curl
...
Dependencies Resolved

================================================================================
 Package                        Arch     Version              Repository   Size
================================================================================
Installing:
 ansible                        noarch   2.7.7-1.el7          epel         11 M
Updating:
 curl                           x86_64   7.29.0-51.el7        base        269 k
Installing for dependencies:
 PyYAML                         x86_64   3.10-11.el7          base        153 k
 libtomcrypt                    x86_64   1.17-26.el7          extras      224 k
 libtommath                     x86_64   0.42.0-6.el7         extras       36 k
 libyaml                        x86_64   0.1.4-11.el7_0       base         55 k
 python-babel                   noarch   0.9.6-8.el7          base        1.4 M
 python-cffi                    x86_64   1.6.0-5.el7          base        218 k
 python-enum34                  noarch   1.0.4-1.el7          base         52 k
 python-httplib2                noarch   0.9.2-1.el7          extras      115 k
 python-idna                    noarch   2.4-1.el7            base         94 k
 python-jinja2                  noarch   2.7.2-2.el7          base        515 k
 python-keyczar                 noarch   0.71c-2.el7          epel        218 k
 python-markupsafe              x86_64   0.11-10.el7          base         25 k
 python-paramiko                noarch   2.1.1-9.el7          updates     269 k
 python-ply                     noarch   3.4-11.el7           base        123 k
 python-pycparser               noarch   2.14-1.el7           base        104 k
 python-six                     noarch   1.9.0-2.el7          base         29 k
 python2-crypto                 x86_64   2.6.1-15.el7         extras      477 k
 python2-cryptography           x86_64   1.7.2-2.el7          base        502 k
 python2-jmespath               noarch   0.9.0-3.el7          extras       39 k
 python2-pyasn1                 noarch   0.1.9-7.el7          base        100 k
Updating for dependencies:
 libcurl                        x86_64   7.29.0-51.el7        base        221 k
 nspr                           x86_64   4.19.0-1.el7_5       base        127 k
 nss                            x86_64   3.36.0-7.1.el7_6     updates     835 k
 nss-pem                        x86_64   1.0.3-5.el7          base         74 k
 nss-softokn                    x86_64   3.36.0-5.el7_5       base        315 k
 nss-softokn-freebl             x86_64   3.36.0-5.el7_5       base        222 k
 nss-sysinit                    x86_64   3.36.0-7.1.el7_6     updates      62 k
 nss-tools                      x86_64   3.36.0-7.1.el7_6     updates     515 k
 nss-util                       x86_64   3.36.0-1.1.el7_6     updates      78 k

Transaction Summary
================================================================================
...

# ansible --version
ansible 2.7.7
  config file = /etc/ansible/ansible.cfg
  configured module search path = [u'/root/.ansible/plugins/modules', u'/usr/share/ansible/plugins/modules']
  ansible python module location = /usr/lib/python2.7/site-packages/ansible
  executable location = /usr/bin/ansible
  python version = 2.7.5 (default, Aug  4 2017, 00:39:18) [GCC 4.8.5 20150623 (Red Hat 4.8.5-16)]

# su - tidb
$ git clone https://github.com/pingcap/thirdparty-ops.git

-- 编辑配置文件：
$cd /home/tidb/thirdparty-ops/kafka-ansible
$$ vim inventory.ini


[zookeeper_servers]
zk1 ansible_host=172.16.4.173  deploy_dir=/data/tidb/zk_deploy  myid=1
zk2 ansible_host=172.16.4.174  deploy_dir=/data/tidb/zk_deploy  myid=2
zk3 ansible_host=172.16.4.175  deploy_dir=/data/tidb/zk_deploy  myid=3

# deploy_dir: zookeeper deployment directory
# myid: zookeeper myid, it must be unique and range 1-255

[kafka_servers]
kafka1 ansible_host=172.16.4.173 deploy_dir=/data/tidb/kafka_deploy data_dirs=/data/tidb/kafka_data kafka_port=9092  id=1
kafka2 ansible_host=172.16.4.174 deploy_dir=/data/tidb/kafka_deploy data_dirs=/data/tidb/kafka_data kafka_port=9092  id=2
kafka3 ansible_host=172.16.4.175 deploy_dir=/data/tidb/kafka_deploy data_dirs=/data/tidb/kafka_data kafka_port=9092  id=3

# deploy_dir: Kafka deployment directory
# kafka_port: Kafka port
# data_dirs: Kakfa's data directory
# id: id of the broker, it must be set to a unique integer for each broker

## Global variables
[all:vars]
# ssh via normal user
ansible_user = tidb

# KAFKA_HEAP_OPTS
kafka_heap_size = "3G"

# timezone of deployment region
timezone = Asia/Shanghai
set_timezone = True

enable_ntpd = True
set_hostname = False
enable_firewalld = False

说明：


-- 下载软件：
$ pwd
/home/tidb/thirdparty-ops/kafka-ansible
$ ansible-playbook -i inventory.ini local_prepare.yml
此步骤依赖的软件包： kafka-ansible/roles/packages/packagesfiles

-- 初始化远程主机，设置环境变量：
$ ansible-playbook -i inventory.ini bootstrap.yml

-- 安装部署依赖的包和 zookeeper、kafka：
$ ansible-playbook -i inventory.ini deploy.yml
-- 启动zookeeper和kafka：
$ ansible-playbook -i inventory.ini start.yml

PLAY RECAP *****************************************************************************************
kafka1                     : ok=13   changed=1    unreachable=0    failed=0   
kafka2                     : ok=12   changed=1    unreachable=0    failed=0   
kafka3                     : ok=12   changed=1    unreachable=0    failed=0   
localhost                  : ok=1    changed=0    unreachable=0    failed=0   
zk1                        : ok=13   changed=2    unreachable=0    failed=0   
zk2                        : ok=13   changed=2    unreachable=0    failed=0   
zk3                        : ok=13   changed=2    unreachable=0    failed=0 

--登录到部署的主机查看进程：
# jps
25916 QuorumPeerMain
26653 Jps
26430 Kafka
可以看到有kafka和zookeeper的进程。

-- 关闭zookeeper和kafka：
$ ansible-playbook -i inventory.ini stop.yml


-- zookeeper和kafka扩展：
1.编辑文件：$ vim inventory.ini
2.执行下列几个步骤：
ansible-playbook -i inventory.ini prepare.yml --diff
ansible-playbook -i inventory.ini deploy.yml --diff
ansible-playbook -i inventory.ini start.yml --diff
-- 手动启动管你zookeeper和kafka：
在zk和kafka的主机上执行命令：
cd $deploy_dir/scripts && ./run_zookeeper.sh start|status|stop
cd $deploy_dir/scripts && ./run_kafka.sh start|stop


--生产者和消费者测试：
在中控机上执行：
$cd /home/tidb/thirdparty-ops/kafka-ansible
1.生产者：
tools/kafka-console-consumer -brokers="172.17.8.201:9092,172.17.8.202:9092,172.17.8.203:9092" -topic=test

2.消费者：
tools/kafka-console-producer -brokers="172.17.8.201:9092,172.17.8.202:9092,172.17.8.203:9092" -topic=test -value=world -key=hello

参考：

https://github.com/pingcap/thirdparty-ops/tree/master/kafka-ansible