kubernetes上部署kafka集群
简介
参考官方文档
http://kafka.apache.org/intro
准备
Host: centos
Ip: 192.168.122.144
关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
关闭selinux
setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
简单的说，要在kubernets集群上运行kafka集群你需要做以下步骤：
1.安装一个kubernetes集群
2.安装一个本地存储，例如NFS作为kubernetes的后台服务
3.创建一个storage class
4.部署zookeeper集群
5.部署kafka集群
部署 kubernetes
可以使用熟悉的工具，例如：kubeadm
https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
安装NFS
yum -y install nfs-utils.x86_64
mkdir /opt/data
cat /etc/exports
/opt/data/ 192.168.122.0/24(rw,sync,no_root_squash)
systemctl restart nfs
systemctl enable nfs
在kubernetes集群上部署nfs-client
git clone https://github.com/kubernetes-incubator/external-storage.git
cd external-storage/nfs-client
设置授权
如果你的kubernetes集群启用了RBAC，你必须进行授权，如果你的名字空间不是在default下，请编辑deploy/deployment。
NS=$(kubectl config get-contexts|grep -e "^\*" |awk '{print $5}')
NAMESPACE=${NS:-default}
sed -i'' "s/namespace:.*/namespace: $NAMESPACE/g" ./deploy/rbac.yaml ./deploy/deployment.yaml
kubectl create -f deploy/rbac.yaml
配置NFS-Client provisioner
编辑deploy/deploument.yaml替换你的NFS的信息
...
env:
- name: PROVISIONER_NAME
value: fuseim.pri/ifs
- name: NFS_SERVER
value:
- name: NFS_PATH
value: /var/nfs
volumes:
- name: nfs-client-root
nfs:
server:
path: /var/nfs
kubectl apply -f deploy/deployment.yaml
创建storage class
kubectl apply -f deploy/class.yaml
[root@localhost ~]# kubectl get sc
NAME PROVISIONER AGE
managed-nfs-storage fuseim.pri/ifs 22h
部署zookeeper
创建configmap
cat > zookeeper-config.yaml << EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: zk-config
data:
ensemble: "zk-0;zk-1;zk-2"
jvm.heap: "512M"
tick: "2000"
init: "10"
sync: "5"
client.cnxns: "60"
snap.retain: "3"
purge.interval: "1"
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: zk-budget
spec:
selector:
matchLabels:
app: zk
minAvailable: 2
EOF
kubectl apply -f zookeeper-config.yaml
创建service
cat > zookeeper-svc.yaml << EOF
apiVersion: v1
kind: Service
metadata:
name: zk-headless
labels:
app: zk-headless
spec:
type: ClusterIP
ports:
- port: 2181
name: client
- port: 2888
name: server
- port: 3888
name: leader-election
selector:
app: zk
EOF
kubectl apply -f zookeeper-svc.yaml
部署zookeeper statefulset
cat > zookeeper-sts.yaml << EOF
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: zk
spec:
serviceName: zk-headless
replicas: 3
template:
metadata:
labels:
app: zk
annotations:
pod.alpha.kubernetes.io/initialized: "true"
spec:
containers:
- name: k8szk
imagePullPolicy: IfNotPresent
image: gcr.io/google_samples/k8szk:v1
ports:
- containerPort: 2181
name: client
- containerPort: 2888
name: server
- containerPort: 3888
name: leader-election
env:
- name : ZK_ENSEMBLE
valueFrom:
configMapKeyRef:
name: zk-config
key: ensemble
- name : ZK_HEAP_SIZE
valueFrom:
configMapKeyRef:
name: zk-config
key: jvm.heap
- name : ZK_TICK_TIME
valueFrom:
configMapKeyRef:
name: zk-config
key: tick
- name : ZK_INIT_LIMIT
valueFrom:
configMapKeyRef:
name: zk-config
key: init
- name : ZK_SYNC_LIMIT
valueFrom:
configMapKeyRef:
name: zk-config
key: tick
- name : ZK_MAX_CLIENT_CNXNS
valueFrom:
configMapKeyRef:
name: zk-config
key: client.cnxns
- name: ZK_SNAP_RETAIN_COUNT
valueFrom:
configMapKeyRef:
name: zk-config
key: snap.retain
- name: ZK_PURGE_INTERVAL
valueFrom:
configMapKeyRef:
name: zk-config
key: purge.interval
- name: ZK_CLIENT_PORT
value: "2181"
- name: ZK_SERVER_PORT
value: "2888"
- name: ZK_ELECTION_PORT
value: "3888"
command:
- sh
- -c
- zkGenConfig.sh && zkServer.sh start-foreground
readinessProbe:
exec:
command:
- "zkOk.sh"
initialDelaySeconds: 15
timeoutSeconds: 5
livenessProbe:
exec:
command:
- "zkOk.sh"
initialDelaySeconds: 15
timeoutSeconds: 5
volumeMounts:
- name: datadir
mountPath: /var/lib/zookeeper
securityContext:
runAsUser: 1000
fsGroup: 1000
volumeClaimTemplates:
- metadata:
name: datadir
spec:
storageClassName: portworx-sc
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 2Gi
EOF
kubectl apply -f zookeeper-sts.yaml
确认zookeeper pod是否全部running
[root@localhost deploy]# kubectl get pod
NAME READY STATUS RESTARTS AGE
zk-0 1/1 Running 0 5m
zk-1 1/1 Running 0 4m
zk-2 1/1 Running 0 3m
测试zookeeper集群的功能是否正常，通过从不同节点的创建、检索值来检验。
首先在zk-0节点创建key/value
[root@localhost deploy]# kubectl exec zk-0 -- /opt/zookeeper/bin/zkCli.sh create /hello world
Connecting to localhost:2181
2019-11-13 06:52:09,125 [myid:] - INFO [main:Environment@100] - Client environment:zookeeper.version=3.4.9-1757313, built on 08/23/2016 06:50 GMT
2019-11-13 06:52:09,130 [myid:] - INFO [main:Environment@100] - Client environment:host.name=zk-0.zk-headless.default.svc.cluster.local
2019-11-13 06:52:09,130 [myid:] - INFO [main:Environment@100] - Client environment:java.version=1.8.0_111
2019-11-13 06:52:09,133 [myid:] - INFO [main:Environment@100] - Client environment:java.vendor=Oracle Corporation
2019-11-13 06:52:09,133 [myid:] - INFO [main:Environment@100] - Client environment:java.home=/usr/lib/jvm/java-8-openjdk-amd64/jre
2019-11-13 06:52:09,133 [myid:] - INFO [main:Environment@100] - Client environment:java.class.path=/opt/zookeeper/bin/../build/classes:/opt/zookeeper/bin/../build/lib/*.jar:/opt/zookeeper/bin/../lib/slf4j-log4j12-1.6.1.jar:/opt/zookeeper/bin/../lib/slf4j-api-1.6.1.jar:/opt/zookeeper/bin/../lib/netty-3.10.5.Final.jar:/opt/zookeeper/bin/../lib/log4j-1.2.16.jar:/opt/zookeeper/bin/../lib/jline-0.9.94.jar:/opt/zookeeper/bin/../zookeeper-3.4.9.jar:/opt/zookeeper/bin/../src/java/lib/*.jar:/opt/zookeeper/bin/../conf:
2019-11-13 06:52:09,134 [myid:] - INFO [main:Environment@100] - Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib
2019-11-13 06:52:09,134 [myid:] - INFO [main:Environment@100] - Client environment:java.io.tmpdir=/tmp
2019-11-13 06:52:09,134 [myid:] - INFO [main:Environment@100] - Client environment:java.compiler=
2019-11-13 06:52:09,134 [myid:] - INFO [main:Environment@100] - Client environment:os.name=Linux
2019-11-13 06:52:09,134 [myid:] - INFO [main:Environment@100] - Client environment:os.arch=amd64
2019-11-13 06:52:09,134 [myid:] - INFO [main:Environment@100] - Client environment:os.version=3.10.0-957.el7.x86_64
2019-11-13 06:52:09,134 [myid:] - INFO [main:Environment@100] - Client environment:user.name=zookeeper
2019-11-13 06:52:09,134 [myid:] - INFO [main:Environment@100] - Client environment:user.home=/home/zookeeper
2019-11-13 06:52:09,135 [myid:] - INFO [main:Environment@100] - Client environment:user.dir=/
2019-11-13 06:52:09,136 [myid:] - INFO [main:ZooKeeper@438] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@1de0aca6
2019-11-13 06:52:09,166 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1032] - Opening socket connection to server localhost/127.0.0.1:2181. Will not attempt to authenticate using SASL (unknown error)
2019-11-13 06:52:09,263 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@876] - Socket connection established to localhost/127.0.0.1:2181, initiating session
2019-11-13 06:52:09,416 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1299] - Session establishment complete on server localhost/127.0.0.1:2181, sessionid = 0x16e6232c5210001, negotiated timeout = 30000
WATCHER::
WatchedEvent state:SyncConnected type:None path:null
Node already exists: /hello
在zk-2节点上去检索这个值
[root@localhost deploy]# kubectl exec zk-2 -- /opt/zookeeper/bin/zkCli.sh get /hello
Connecting to localhost:2181
2019-11-13 06:56:26,032 [myid:] - INFO [main:Environment@100] - Client environment:zookeeper.version=3.4.9-1757313, built on 08/23/2016 06:50 GMT
2019-11-13 06:56:26,036 [myid:] - INFO [main:Environment@100] - Client environment:host.name=zk-2.zk-headless.default.svc.cluster.local
2019-11-13 06:56:26,036 [myid:] - INFO [main:Environment@100] - Client environment:java.version=1.8.0_111
2019-11-13 06:56:26,039 [myid:] - INFO [main:Environment@100] - Client environment:java.vendor=Oracle Corporation
2019-11-13 06:56:26,039 [myid:] - INFO [main:Environment@100] - Client environment:java.home=/usr/lib/jvm/java-8-openjdk-amd64/jre
2019-11-13 06:56:26,039 [myid:] - INFO [main:Environment@100] - Client environment:java.class.path=/opt/zookeeper/bin/../build/classes:/opt/zookeeper/bin/../build/lib/*.jar:/opt/zookeeper/bin/../lib/slf4j-log4j12-1.6.1.jar:/opt/zookeeper/bin/../lib/slf4j-api-1.6.1.jar:/opt/zookeeper/bin/../lib/netty-3.10.5.Final.jar:/opt/zookeeper/bin/../lib/log4j-1.2.16.jar:/opt/zookeeper/bin/../lib/jline-0.9.94.jar:/opt/zookeeper/bin/../zookeeper-3.4.9.jar:/opt/zookeeper/bin/../src/java/lib/*.jar:/opt/zookeeper/bin/../conf:
2019-11-13 06:56:26,039 [myid:] - INFO [main:Environment@100] - Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib/x86_64-linux-gnu/jni:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/lib/jni:/lib:/usr/lib
2019-11-13 06:56:26,039 [myid:] - INFO [main:Environment@100] - Client environment:java.io.tmpdir=/tmp
2019-11-13 06:56:26,040 [myid:] - INFO [main:Environment@100] - Client environment:java.compiler=
2019-11-13 06:56:26,040 [myid:] - INFO [main:Environment@100] - Client environment:os.name=Linux
2019-11-13 06:56:26,040 [myid:] - INFO [main:Environment@100] - Client environment:os.arch=amd64
2019-11-13 06:56:26,040 [myid:] - INFO [main:Environment@100] - Client environment:os.version=3.10.0-957.el7.x86_64
2019-11-13 06:56:26,040 [myid:] - INFO [main:Environment@100] - Client environment:user.name=zookeeper
2019-11-13 06:56:26,040 [myid:] - INFO [main:Environment@100] - Client environment:user.home=/home/zookeeper
2019-11-13 06:56:26,041 [myid:] - INFO [main:Environment@100] - Client environment:user.dir=/
2019-11-13 06:56:26,042 [myid:] - INFO [main:ZooKeeper@438] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@1de0aca6
2019-11-13 06:56:26,076 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1032] - Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error)
2019-11-13 06:56:26,171 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@876] - Socket connection established to localhost/0:0:0:0:0:0:0:1:2181, initiating session
2019-11-13 06:56:26,361 [myid:] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1299] - Session establishment complete on server localhost/0:0:0:0:0:0:0:1:2181, sessionid = 0x36e6232cbca0001, negotiated timeout = 30000
WATCHER::
WatchedEvent state:SyncConnected type:None path:null
world
cZxid = 0x100000002
ctime = Mon Nov 11 08:37:02 UTC 2019
mZxid = 0x100000002
mtime = Mon Nov 11 08:37:02 UTC 2019
pZxid = 0x100000002
cversion = 0
dataVersion = 0
aclVersion = 0
ephemeralOwner = 0x0
dataLength = 5
numChildren = 0
部署kafka statefulset
创建configmap
cat > kafka-config.yaml << EOF
apiVersion: v1
kind: ConfigMap
metadata:
name: broker-config
namespace: default
data:
init.sh: |-
#!/bin/bash
set -x KAFKA_BROKER_ID=${HOSTNAME##*-}
cp -Lur /etc/kafka-configmap/* /etc/kafka/
sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties
hash kubectl 2>/dev/null || {
sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/" /etc/kafka/server.properties
} && {
ZONE=$(kubectl get node "$NODE_NAME" -o=go-template={{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}})
if [ $? -ne 0 ]; then
sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone lookup failed, see -c init-config logs/" /etc/kafka/server.properties
elif [ "x$ZONE" == "x" ]; then
sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/" /etc/kafka/server.properties
else
sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties
fi
}
server.properties: |-
delete.topic.enable=true
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/tmp/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=zk-0.zk-headless.default.svc.cluster.local:2181,zk-1.zk-headless.default.svc.cluster.local:2181,zk-2.zk-headless.default.svc.cluster.local:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0
log4j.properties: |-
log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH
log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH
log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log
log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH
log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log
log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH
log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log
log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH
log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log
log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender
log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH
log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log
log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
# Change the two lines below to adjust ZK client logging
log4j.logger.org.I0Itec.zkclient.ZkClient=INFO
log4j.logger.org.apache.zookeeper=INFO
# Change the two lines below to adjust the general broker logging level (output to server.log and stdout)
log4j.logger.kafka=INFO
log4j.logger.org.apache.kafka=INFO
# Change to DEBUG or TRACE to enable request logging
log4j.logger.kafka.request.logger=WARN, requestAppender
log4j.additivity.kafka.request.logger=false
log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender
log4j.additivity.kafka.network.RequestChannel$=false
log4j.logger.kafka.controller=TRACE, controllerAppender
log4j.additivity.kafka.controller=false
log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender
log4j.additivity.kafka.log.LogCleaner=false
log4j.logger.state.change.logger=TRACE, stateChangeAppender
log4j.additivity.state.change.logger=false
log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender
log4j.additivity.kafka.authorizer.logger=false
EOF
kubectl apply -f kafka-config.yaml
创建service
cat > kafka-svc.yaml << EOF
apiVersion: v1
kind: Service
metadata:
name: kafka-broker
namespace: default
spec:
type: ClusterIP
ports:
- port: 9092
# [podname].broker.kafka.svc.cluster.local
selector:
app: kafka
---
EOF
kubectl apply -f kafka-svc.yaml
部署kafka集群
cat > kafka-sts.yaml << EOF
apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
name: kafka
namespace: default
spec:
serviceName: "kafka-broker"
replicas: 1
template:
metadata:
labels:
app: kafka
annotations:
spec
initContainers:
- name: init-config
image: solsson/kafka-initutils@sha256:c275d681019a0d8f01295dbd4a5bae3cfa945c8d0f7f685ae1f00f2579f08c7d
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
command: ['/bin/bash', '/etc/kafka-configmap/init.sh']
volumeMounts:
- name: configmap
mountPath: /etc/kafka-configmap
- name: config
mountPath: /etc/kafka
containers:
- name: broker
image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
env:
- name: KAFKA_LOG4J_OPTS
value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
ports:
- containerPort: 9092
command:
- ./bin/kafka-server-start.sh
- /etc/kafka/server.properties
- --override
- zookeeper.connect=zk-0.zk-headless.default.svc.cluster.local:2181,zk-1.zk-headless.default.svc.cluster.local:2181,zk-2.zk-headless.default.svc.cluster.local:2181
- --override
- log.retention.hours=-1
- --override
- log.dirs=/var/lib/kafka/data/topics
- --override
- auto.create.topics.enable=false
resources:
requests:
cpu: 100m
memory: 512Mi
readinessProbe:
exec:
command:
- /bin/sh
- -c
- 'echo "" | nc -w 1 127.0.0.1 9092'
volumeMounts:
- name: config
mountPath: /etc/kafka
- name: data
mountPath: /var/lib/kafka/data
volumes:
- name: configmap
configMap:
name: broker-config
- name: config
emptyDir: {}
volumeClaimTemplates:
- metadata:
name: data
spec:
storageClassName: portworx-sc-rep3
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 3Gi
EOF
kubectl apply -f kafka-sts.yaml
启动一个redis cli去连接redis集群
cat > kafka-cli.yaml << EOF
apiVersion: v1
kind: Pod
metadata:
name: kafka-cli
spec:
containers:
- name: kafka
image: solsson/kafka:0.11.0.0
command:
- sh
- -c
- "exec tail -f /dev/null"
EOF
kubectl create -f kafka-cli.yaml
测试
[root@localhost deploy]# kubectl exec -it kafka-cli bash
root@kafka-cli:/opt/kafka# ./bin/kafka-topics.sh --create --zookeeper zk-headless:2181 --replication-factor 1 --partitions 1 --topic my-test
Created topic "my-test".
root@kafka-cli:/opt/kafka# ./bin/kafka-console-producer.sh --broker-list kafka-broker:9092 --topic my-test
>my test message
>message test
输入最后一条消息后按ctrl+c，然后执行下面步骤。
root@kafka-cli:/opt/kafka# ./bin/kafka-console-consumer.sh --bootstrap-server kafka-broker:9092 --topic my-test --partition 0 --from-beginning
my test message
message test
测试成功，集群部署完毕
参考：
https://github.com/kubernetes-incubator/external-storage/tree/master/nfs-client
https://portworx.com/run-ha-kafka-cluster-ibm-cloud-kubernetes-service/