docker搭建zk+kafka+flume+es+kibana日志处理系统
使用三台服务器
zookeeper集群搭建
镜像获取
docker pull zookeeper:latest
每台服务器创建挂载目录
mkdir -p /home/docker/zk-test/data
mkdir -p /home/docker/zk-test/logs
分别启动三台服务器zk
docker run --name zk-test --net=host --restart always -v /home/docker/zk-test:/data/zookeeper -e ZOO_PORT=2181 -e ZOO_DATA_DIR=/data/zookeeper/data -e ZOO_DATA_LOG_DIR=/data/zookeeper/logs -e ZOO_MY_ID=1 -e ZOO_SERVERS="server.1=192.168.0.100:2888:3888 server.2=192.168.0.101:2888:3888 server.3=192.168.0.102:2888:3888" -d zookeeper:latest
docker run --name zk-test --net=host --restart always -v /home/docker/zk-test:/data/zookeeper -e ZOO_PORT=2181 -e ZOO_DATA_DIR=/data/zookeeper/data -e ZOO_DATA_LOG_DIR=/data/zookeeper/logs -e ZOO_MY_ID=2 -e ZOO_SERVERS="server.1=192.168.0.100:2888:3888 server.2=192.168.0.101:2888:3888 server.3=192.168.0.102:2888:3888" -d zookeeper:latest
docker run --name zk-test --net=host --restart always -v /home/docker/zk-test:/data/zookeeper -e ZOO_PORT=2181 -e ZOO_DATA_DIR=/data/zookeeper/data -e ZOO_DATA_LOG_DIR=/data/zookeeper/logs -e ZOO_MY_ID=3 -e ZOO_SERVERS="server.1=192.168.0.100:2888:3888 server.2=192.168.0.101:2888:3888 server.3=192.168.0.102:2888:3888" -d zookeeper:latest
- 其它都一样，注意
ZOO_MY_ID=[1,2,3]参数。
验证
- 进入其中一个容器：
docker exec -it zk-test /bin/bash
- 执行：
zkServer.sh status
如下结果即为成功：
bash-4.4# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /conf/zoo.cfg
Mode: follower
kakfa集群搭建
镜像获取
docker pull wurstmeister/kafka:latest
分别启动三台服务器kafka
docker run --name kafka-test --net=host --restart always --volume /home/docker/kafka-test/data:/data -e KAFKA_BROKER_ID=1 -e KAFKA_PORT=9092 -e KAFKA_HEAP_OPTS="-Xms2g -Xmx4g" -e KAFKA_HOST_NAME=192.168.0.100 -e KAFKA_ADVERTISED_HOST_NAME=192.168.0.100 -e KAFKA_LOG_DIRS=/data/kafka -e KAFKA_ZOOKEEPER_CONNECT="192.168.0.100:2181,192.168.0.101:2181,192.168.0.102:2181" -d wurstmeister/kafka:latest
docker run --name kafka-test --net=host --restart always --volume /home/docker/kafka-test/data:/data -e KAFKA_BROKER_ID=2 -e KAFKA_PORT=9092 -e KAFKA_HEAP_OPTS="-Xms2g -Xmx4g" -e KAFKA_HOST_NAME=192.168.0.101 -e KAFKA_ADVERTISED_HOST_NAME=192.168.0.101 -e KAFKA_LOG_DIRS=/data/kafka -e KAFKA_ZOOKEEPER_CONNECT="192.168.0.100:2181,192.168.0.101:2181,192.168.0.102:2181" -d wurstmeister/kafka:latest
docker run --name kafka-test --net=host --restart always --volume /home/docker/kafka-test/data:/data -e KAFKA_BROKER_ID=3 -e KAFKA_PORT=9092 -e KAFKA_HEAP_OPTS="-Xms2g -Xmx4g" -e KAFKA_HOST_NAME=192.168.0.102 -e KAFKA_ADVERTISED_HOST_NAME=192.168.0.102 -e KAFKA_LOG_DIRS=/data/kafka -e KAFKA_ZOOKEEPER_CONNECT="192.168.0.100:2181,192.168.0.101:2181,192.168.0.102:2181" -d wurstmeister/kafka:latest
验证
- 随便一台服务器进入容器
docker exec -it kafka-test /bin/bash
- 创建topic
/opt/kafka/bin/kafka-topics.sh --create --zookeeper 192.168.0.100:2181,192.168.0.101:2181,192.168.0.102:2181 --topic test --partitions 3 --replication-factor 2
- 开启消费
/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server 192.168.0.101:9092 --topic test --from-beginning
- 重新开启终端进入容器后生产数据
/opt/kafka/bin/kafka-console-producer.sh --broker-list 192.168.0.101:9092 --topic test
- 只要能消费到数据即集群创建成功
flume安装
镜像获取
docker pull probablyfine/flume:latest
创建挂在目录存放flume.conf自定义配置
mkdir -p /home/docker/flume-test
cd home/docker/flume-test
vim flume.conf
复制以下内容到flume.conf中
# Name the components on this agent
datahub101.sources = r1 r2 r3
datahub101.sinks = k1
datahub101.channels = c1
# Describe/configure the source
datahub101.sources.r1.type = exec
datahub101.sources.r1.command = tail -F /var/tmp/flume_log/test1/test1.log
datahub101.sources.r1.interceptors = i1
datahub101.sources.r1.interceptors.i1.type = static
datahub101.sources.r1.interceptors.i1.key = type
datahub101.sources.r1.interceptors.i1.value = test1
datahub101.sources.r2.type = exec
datahub101.sources.r2.command = tail -F /var/tmp/flume_log/test2/test2.log
datahub101.sources.r2.interceptors = i2
datahub101.sources.r2.interceptors.i2.type = static
datahub101.sources.r2.interceptors.i2.key = type
datahub101.sources.r2.interceptors.i2.value = test2
datahub101.sources.r3.type = exec
datahub101.sources.r3.command = tail -F /var/tmp/flume_log/test3/test3.log
datahub101.sources.r3.interceptors = i3
datahub101.sources.r3.interceptors.i3.type = static
datahub101.sources.r3.interceptors.i3.key = type
datahub101.sources.r3.interceptors.i3.value = test3
# Describe the sink
datahub101.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
datahub101.sinks.k1.kafka.topic =test
datahub101.sinks.k1.kafka.bootstrap.servers = 192.168.0.101:9092
datahub101.sinks.k1.kafka.flumeBatchSize = 20
datahub101.sinks.k1.kafka.producer.acks = 1
datahub101.sinks.k1.kafka.producer.linger.ms = 1
datahub101.sinks.ki.kafka.producer.compression.type = snappy
# Use a channel which buffers events in memory
datahub101.channels.c1.type = memory
datahub101.channels.c1.capacity = 1000
datahub101.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
datahub101.sources.r1.channels = c1
datahub101.sources.r2.channels = c1
datahub101.sources.r3.channels = c1
datahub101.sinks.k1.channel = c1
启动flume
docker run --name flume-test --restart always --net=host -v /home/docker/flume-test/flume.conf:/opt/flume-config/flume.conf -v /home/docker/flume-test/flume_log:/var/tmp/flume_log -e FLUME_AGENT_NAME="AGENT-101" -d probablyfine/flume:latest
验证
- 开启kafka消费test topic。
echo "test msg" >> test1.log
ElasticSearch安装
镜像获取
docker pull docker.elastic.co/elasticsearch/elasticsearch:7.0.0
创建挂载目录及启动参数
mkdir -p /home/docker/es-test/data
chmod 755 /home/docker/es-test/data
cd /home/docker/es-test
vim elasticsearch.yml
将下列参数加到elasticsearch.yml中
#集群名称，多个节点用一个名称
cluster.name: test
# es 1.0版本的默认配置是 "0.0.0.0"，所以不绑定ip也可访问
network.host: 0.0.0.0
node.name: master
#跨域设置
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: true
node.data: true
镜像运行
docker run --name es-test -p 9200:9200 -p 9300:9300 -p 5601:5601 -e "discovery.type=single-node" -v /home/docker/es-test/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /home/docker/es-test/data:/usr/share/elasticsearch/data -d docker.elastic.co/elasticsearch/elasticsearch:7.0.0
kibana安装与es同服务器
镜像获取
docker pull docker.elastic.co/kibana/kibana:7.0.0
启动
docker run -d -e ELASTICSEARCH_URL=http://192.168.0.100:9200 --name kibana-test --network=container:es-test docker.elastic.co/kibana/kibana:7.0.0
- 这里没有把配置文件挂载出来，需要进入容器中的/config/kibana.yml用下方配置替换一下，再重启容器，es的host用的是服务es-test的容器内的ip，可以
docker logs es-test查看es启动的容器内的ip。
server.name: kibana
server.host: 0.0.0.0
elasticsearch.hosts: [ "http://172.17.0.2:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true
验证
访问192.168.0.100:5601即可。