在Kubernetes上搭建NFS的STORAGECLASS动态分配置持久卷PV
原来使用Kubernetes时，用的比较多的是静态存储或用第三方的GCE，AWS等动态共享存储。
搭建Glusterfs也比较烦锁，最后查看有同学使用了nfs做了动态存储。
首先来了解一下PV,PVC,STORAGECLASS
Persistent Volume 简称PV是一个K8S资源对象，所以我们可以单独创建一个PV。它不和Pod直接发生关系，而是通过Persistent Volume Claim，简称PVC来实现动态绑定。Pod定义里指定的是PVC，然后PVC会根据Pod的要求去自动绑定合适的PV给Pod使用。
比如，一个配置了许多50Gi PV的集群不会匹配到一个要求100Gi的PVC。 只有在100Gi PV被加到集群之后，这个PVC才可以被绑定。
创建PV有两种方式，静态和动态。
静态，是管理员手动创建一堆PV，组成一个PV池，供PVC来绑定。
动态，是指在现有PV不满足PVC的请求时，可以使用存储分类(StorageClass)，描述具体过程为：PV先创建分类，PVC请求已创建的某个类（StorageClass）的资源，这样就达到动态配置的效果。即通过一个叫 Storage Class的对象由存储系统根据PVC的要求自动创建。
其中动态方式是通过StorageClass来完成的，这是一种新的存储供应方式。
动态卷供给能力让管理员不必进行预先创建存储卷，而是随用户需求进行创建。
使用StorageClass有什么好处呢？除了由存储系统动态创建，节省了管理员的时间，还有一个好处是可以封装不同类型的存储供PVC选用。在StorageClass出现以前，PVC绑定一个PV只能根据两个条件，一个是存储的大小，另一个是访问模式。在StorageClass出现后，等于增加了一个绑定维度。
下面通过nfs的创建一个共享存储进行动态分配卷：
1、定义nfs驱动篇排：
nfs-deployment.yml
{
"kind": "Deployment",
"apiVersion": "extensions/v1beta1",
"metadata": {
"name": "nfs-kafka-provisioner",
"namespace": "kube-system",
"labels": {
"app": "nfs-kafka-provisioner"
},
"annotations": {
"deployment.kubernetes.io/revision": "1"
}
},
"spec": {
"replicas": 1,
"selector": {
"matchLabels": {
"app": "nfs-kafka-provisioner"
}
},
"template": {
"metadata":
"labels": {
"app": "nfs-kafka-provisioner"
}
},
"spec": {
"volumes": [
{
"name": "nfs-kafka-root",
"nfs": {
"server": "192.168.42.23", #使用nfs创建好的共享路径
"path": "/data/kafka"
}
}
],
"containers": [
{
"name": "nfs-kafaka-provisioner",
"image": "registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner",
"env": [
{
"name": "PROVISIONER_NAME", #指定provisioner为k8s.io/nfs-provisioner-kafka
"value": "k8s.io/nfs-provisioner-kafka"
},
{
"name": "NFS_SERVER",
"value": "192.168.42.23"
},
{
"name": "NFS_PATH",
"value": "/data/kafka"
}
],
"resources": {},
"volumeMounts": [
{
"name": "nfs-kafka-root",
"mountPath": "/persistentvolumes"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "Always"
}
],
"restartPolicy": "Always",
"terminationGracePeriodSeconds": 30,
"dnsPolicy": "ClusterFirst",
"securityContext": {},
"schedulerName": "default-scheduler"
}
},
"strategy": {
"type": "Recreate"
},
"revisionHistoryLimit": 10,
"progressDeadlineSeconds": 600
}
}
2、定义storageclass的篇排文件
nfs-storage.yml:
{
"kind": "StorageClass",
"apiVersion": "storage.k8s.io/v1",
"metadata": {
"name": "managed-nfs-storage-kafka"
},
"provisioner": "k8s.io/nfs-provisioner-kafka" 与deployment的env中的provisioner一致
"reclaimPolicy": "Delete",
"volumeBindingMode": "Immediate"
}
3、编排一个Statusfulset,实验nfs的storage是否能动态生成pvc,pv:
kafka-statusfulset.yml:
{
"kind": "StatefulSet",
"apiVersion": "apps/v1beta2",
"metadata": {
"name": "kafka",
"namespace": "kube-system",
"labels": {
"app": "kafka"
}
},
"spec": {
"replicas": 3,
"selector": {
"matchLabels": {
"app": "kafka"
}
},
"template": {
"metadata": {
"creationTimestamp": null,
"labels": {
"app": "kafka"
}
},
"spec": {
"containers": [
{
"name": "k8skafka",
"image": "k8skafka:v1",
"command": [
"sh",
"-c",
"exec kafka-server-start.sh /opt/kafka/config/server.properties --override broker.id=${HOSTNAME##*-} --override listeners=PLAINTEXT://:9093 --override zookeeper.connect=zk-0.zk-svc.kube-system.svc.cluster.local:2181,zk-1.zk-svc.kube-system.svc.cluster.local:2181,zk-2.zk-svc.kube-system.svc.cluster.local:2181 --override log.dir=/var/lib/kafka --override auto.create.topics.enable=true --override auto.leader.rebalance.enable=true --override background.threads=10 --override compression.type=producer --override delete.topic.enable=false --override leader.imbalance.check.interval.seconds=300 --override leader.imbalance.per.broker.percentage=10 --override log.flush.interval.messages=9223372036854775807 --override log.flush.offset.checkpoint.interval.ms=60000 --override log.flush.scheduler.interval.ms=9223372036854775807 --override log.retention.bytes=-1 --override log.retention.hours=168 --override log.roll.hours=168 --override log.roll.jitter.hours=0 --override log.segment.bytes=1073741824 --override log.segment.delete.delay.ms=60000 --override message.max.bytes=1000012 --override min.insync.replicas=1 --override num.io.threads=8 --override num.network.threads=3 --override num.recovery.threads.per.data.dir=1 --override num.replica.fetchers=1 --override offset.metadata.max.bytes=4096 --override offsets.commit.required.acks=-1 --override offsets.commit.timeout.ms=5000 --override offsets.load.buffer.size=5242880 --override offsets.retention.check.interval.ms=600000 --override offsets.retention.minutes=1440 --override offsets.topic.compression.codec=0 --override offsets.topic.num.partitions=50 --override offsets.topic.replication.factor=3 --override offsets.topic.segment.bytes=104857600 --override queued.max.requests=500 --override quota.consumer.default=9223372036854775807 --override quota.producer.default=9223372036854775807 --override replica.fetch.min.bytes=1 --override replica.fetch.wait.max.ms=500 --override replica.high.watermark.checkpoint.interval.ms=5000 --override replica.lag.time.max.ms=10000 --override replica.socket.receive.buffer.bytes=65536 --override replica.socket.timeout.ms=30000 --override request.timeout.ms=30000 --override socket.receive.buffer.bytes=102400 --override socket.request.max.bytes=104857600 --override socket.send.buffer.bytes=102400 --override unclean.leader.election.enable=true --override zookeeper.session.timeout.ms=6000 --override zookeeper.set.acl=false --override broker.id.generation.enable=true --override connections.max.idle.ms=600000 --override controlled.shutdown.enable=true --override controlled.shutdown.max.retries=3 --override controlled.shutdown.retry.backoff.ms=5000 --override controller.socket.timeout.ms=30000 --override default.replication.factor=1 --override fetch.purgatory.purge.interval.requests=1000 --override group.max.session.timeout.ms=300000 --override group.min.session.timeout.ms=6000 --override inter.broker.protocol.version=0.10.2-IV0 --override log.cleaner.backoff.ms=15000 --override log.cleaner.dedupe.buffer.size=134217728 --override log.cleaner.delete.retention.ms=86400000 --override log.cleaner.enable=true --override log.cleaner.io.buffer.load.factor=0.9 --override log.cleaner.io.buffer.size=524288 --override log.cleaner.io.max.bytes.per.second=1.7976931348623157E308 --override log.cleaner.min.cleanable.ratio=0.5 --override log.cleaner.min.compaction.lag.ms=0 --override log.cleaner.threads=1 --override log.cleanup.policy=delete --override log.index.interval.bytes=4096 --override log.index.size.max.bytes=10485760 --override log.message.timestamp.difference.max.ms=9223372036854775807 --override log.message.timestamp.type=CreateTime --override log.preallocate=false --override log.retention.check.interval.ms=300000 --override max.connections.per.ip=2147483647 --override num.partitions=1 --override producer.purgatory.purge.interval.requests=1000 --override replica.fetch.backoff.ms=1000 --override replica.fetch.max.bytes=1048576 --override replica.fetch.response.max.bytes=10485760 --override reserved.broker.max.id=1000 "
],
"ports": [
{
"name": "server",
"containerPort": 9093,
"protocol": "TCP"
}
],
"env": [
{
"name": "KAFKA_HEAP_OPTS",
"value": "-Xmx512M -Xms512M"
},
{
"name": "KAFKA_OPTS",
"value": "-Dlogging.level=INFO"
}
],
"resources": {
"requests": {
"cpu": "500m",
"memory": "1Gi"
}
},
"volumeMounts": [
{
"name": "datadir",
"mountPath": "/var/lib/kafka"
}
],
"readinessProbe": {
"exec": {
"command": [
"sh",
"-c",
"/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server=localhost:9093"
]
},
"timeoutSeconds": 1,
"periodSeconds": 10,
"successThreshold": 1,
"failureThreshold": 3
},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "IfNotPresent"
}
],
"restartPolicy": "Always",
"terminationGracePeriodSeconds": 300,
"dnsPolicy": "ClusterFirst",
"securityContext": {
"runAsUser": 1000,
"fsGroup": 1000
},
"schedulerName": "default-scheduler"
}
},
"volumeClaimTemplates": [
{
"metadata": {
"name": "datadir",
"creationTimestamp": null,
"annotations": {
"volume.beta.kubernetes.io/storage-class": "managed-nfs-storage-kafka" #通过volumeClaimTemplates的annotations引用storageclass。
}
},
"spec": {
"accessModes": [
"ReadWriteOnce"
],
"resources": {
"requests": {
"storage": "10Gi"
}
}
},
"status": {
"phase": "Pending"
}
}
],
"serviceName": "kafka-svc",
"podManagementPolicy": "OrderedReady",
"updateStrategy": {
"type": "OnDelete"
},
"revisionHistoryLimit": 10
}
}
最终效果如下：
