前面我有讲到一些集群搭建以及高可用配置,那么今天我们就来讲下这个kubefed,那么什么是kubefed呢?其实kubefed中文叫做:“联邦“,也就是说把两个或两个以上的k8s集群关联起来统一管理。像公司有跨异地机房的这种k8s集群的话,完全可以采用这个k8s联邦进行统一管理。我这里仅仅演示两个同局域网内的集群!集群如何搭建请参考我的上两篇文章。好了,直接上干货吧。
我这里用的是一台CentOS7 虚拟机作为客户端,配置不用太高,1C2G都够了!磁盘给8G吧。这台客户机的话要必须能够连通两个集群的master。最好配置在一个网段!
执行如下脚本安装kubectl,helm:
cd /etc/yum.repos.d/
mkdir backup
mv ./*.repo backup
cat < /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
EOF
yum clean all && yum makecache
yum -y update
yum -y install kubectl
cd /usr/local/src/
curl -O https://get.helm.sh/helm-v2.16.9-linux-amd64.tar.gz
tar -zxvf helm-v2.16.9-linux-amd64.tar.gz
cp linux-amd64/helm /usr/bin/
安装好以上两个工具后,我们可以创建两个集群的context模版配置。
kubectl config set-cluster cluster1 --server=https://集群一master的ip:6443 --insecure-skip-tls-verify
kubectl config set-context cluster1 --cluster cluster1 --user kubernetes-admin1
kubectl config set-cluster cluster2 --server=https://集群二master的ip:6443 --insecure-skip-tls-verify
kubectl config set-context cluster2 --cluster cluster2 --user kubernetes-admin2
kubectl config set-credentials kubernetes-admin1 --client-certificate=certfile --client-key=keyfile
kubectl config set-credentials kubernetes-admin2 --client-certificate=certfile --client-key=keyfile
去不同的两个集群master节点获取客户端证书数据和密钥数据,复制抓取到的所有内容替换到客户端相同路径文件里面。
cat .kube/config|grep client-certificate-data
cat .kube/config|grep client-key-data
我的客户端完整配置如下:
[root@client00 ~]# cat .kube/config
apiVersion: v1
clusters:
- cluster:
insecure-skip-tls-verify: true
server: https://集群一master的ip:6443
name: cluster1
- cluster:
insecure-skip-tls-verify: true
server: https://集群二master的ip:6443
name: cluster2
contexts:
- context:
cluster: cluster1
user: kubernetes-admin1
name: cluster1
- context:
cluster: cluster2
user: kubernetes-admin2
name: cluster2
current-context: cluster1
kind: Config
preferences: {}
users:
- name: kubernetes-admin1
user:
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJVkc0Q0VqR2prT2d3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TURBNE1EVXdNek13TlRkYUZ3MHlNVEE0TURVd016TXdOVGhhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQTVyRHZzTWREeDNZTW9jUnIKemtPY3laTVBVOEJPSWJvQW1WWUNqdFVhZktSOXYrTitGUERiTEJLckJBcVd1R0pRN3NjK0RzRUZFeVJpaW5vUwpXZ0syNlB4WllPc2pwUGp6cEROS0lJTHozZUg1UGFQRjl6bjdEbUw2eFV2dnlhWU4yckRiTmtnUnZDZG05eEhjCkEwd3hxa3dZeDJ1dFI5c1JXUWlCeDM2NFk2ZmpwSUVOWDA1dHpzVUFKN0FLT05EVUJSbnl2N2k5bHhjSGQ4Q2cKQkJ5TkxXOHpTMzdHa0NqRjdtSnlCTlVXTTZEbmI3TldjeEgwODNZNkRxbGIwMXR4ektacWUrVmZsTXQ3T0tnUQpPZGpQcnk0U09MQXRjSGpNbnpKSlo4TEJXbFpSdWZPVHNMMXNZZE9PM1BGQUI4SEltSUd3R0JueGp4QXY2WVlVCjhzSnNQUUlEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFGVmVHUWdodEd3TnNMVlcyUkp4clIrSWVHcTZnT283b2hDYwoxOTN2dHEyNG1QRGxCcXF1QkFyR2VKVWEzWDAzT0tiVWdEWEc2TTR4R0x3cnJVb0MyNzRXOVRGK1Ntazk2TlNOCmF6KzJnQ0hWcWxYNU9mWitFa0N4WG0vdklTajVKNzJhazFnaXllYzg3K3RZZkJKWmdiUFg3ZTcvNDlPc1hUaVAKaW5HTkVXQndLWklNQlY0aGdxcVp3WFJTbzVVcmtFK000S2pldzJSM0pXbmRmS3hhWGNrMHRoRS82QVUwbEl0YQpsWGJTRFYvWFQ2aE1uVWxFNWZxVzRVdENGMnczYzBDbEZwZC9zV3JxbEg1S1R3NVM2THhuTHV4TWFXbXhQSmNVClJVekJuZW9QUGpIaytYK1gvSTc4UU9iaEthYzR6Qyt1T0tkeVdNQ2pJNFR1WHcvSzhDZz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBNXJEdnNNZER4M1lNb2NScnprT2N5Wk1QVThCT0lib0FtVllDanRVYWZLUjl2K04rCkZQRGJMQktyQkFxV3VHSlE3c2MrRHNFRkV5Umlpbm9TV2dLMjZQeFpZT3NqcFBqenBETktJSUx6M2VINVBhUEYKOXpuN0RtTDZ4VXZ2eWFZTjJyRGJOa2dSdkNkbTl4SGNBMHd4cWt3WXgydXRSOXNSV1FpQngzNjRZNmZqcElFTgpYMDV0enNVQUo3QUtPTkRVQlJueXY3aTlseGNIZDhDZ0JCeU5MVzh6UzM3R2tDakY3bUp5Qk5VV002RG5iN05XCmN4SDA4M1k2RHFsYjAxdHh6S1pxZStWZmxNdDdPS2dRT2RqUHJ5NFNPTEF0Y0hqTW56SkpaOExCV2xaUnVmT1QKc0wxc1lkT08zUEZBQjhISW1JR3dHQm54anhBdjZZWVU4c0pzUFFJREFRQUJBb0lCQURWdkRsWFlFZThGZk8rTApXNTB2TkFYTXRtUWp5dlVzN0M2WUtZZUQvTnFhRHpaL0x4a1NvUEdZY1Z4Q1I1NzF4dUZIN1N2bHNaTTVma21zCjlNczhqYmx4bWhKRTVNQzFtVm1UTjd4SUdNeHdKeGNMOURMTGowZEp6ZVFkTHJGNmNRTVNxL1BxeFpEYlg1SWkKNU44dXF1SlROZm9iQ3N2MkJoS2xVY292bys0UTZLSEE0OUtBaHRlYjA4N1Z6S3FnbWhHdndadk1zc3VUcWQvdgpzU0tlZW9OSHphbGtZSW56RDFibXZpZVJGNjRCck51b25veFdyZjdqTTdDeit1V2VMdHJQM0gxNmdLWHMvbStvCkYwcnJQekFwTm5Mb3pSWGZ2dG5SbjlwanViWGNEU3dsbW11SzJvR2hMQXFUYUVBaGFiejk2blBkM0ZBZWNEWE4Ka1pEM3lnRUNnWUVBN1g1d2dUZ292MlE2cldQUUZndFhrMnNOdEFHckpYV1RXRDBBZ3ZFUXU1dXYwQlhaZVV4NQp4MVJkaDdjL2pJUnpoMHFSVSt0L2l1MFJpNWhOTnNWYlRvcjR5SW9YMTU4TS94WTdsd2xqZW03Z0piYnVRVzhMCnJLZmdENFFCZVJnV3owZkdPbDVrbi90WXFya2ppaFJGYjVkdXZEZzZLL3poZW01em5ZOWgwSjBDZ1lFQStLckwKY2RyWTVISjBpMkpmUVFFUXY5clg2UU1GM0VJT0E4S1ZNVmNNclM0a1NaK0U0OG9oNjNwcUJBaGl3Vm9GLysyNQp3RmFDaGJrenUwZmpNMlZvSmhLdXAvNDdZZ3F5RVJHbTEzR1VRcVNiUU5PSE8vYjAxaVE5V29ub3ZqYnorVTFkCnZXaW5XZndiQVo5MlBSTmRhN2Q3N09WM3YvQk9hSUlPeEZCbUtDRUNnWUJxRWF5QnFlbFpBWlRMVitiWjBacjkKZGpTYWpwdzNrTG1NRVVLbk9LODJleVFjbFlXWmpUUmVlQ21Gc2F3cVFBV3hDUGE3T2xTdVlZOFNmNlVNcnp4RwphZjhPNGM0Z09TTCswcGt1cmJ6R003cmNYVVV2NkQ2WWtDbWNCR0w4Q0d3M2kyRFhOTTFRUGx1dGRCNlJUdGhzCkxJQUNoRUpHM3h4OUo5TnZMVmlNYVFLQmdRQ0tlaEp4aFAzanVobE14YXRsOThIc0ZXTEhUL2VqZ09WdEVwOGkKcXVkVVNhM0xraG5mQUdHRFVOaFpCcEo0T2ptOUhCV2xGb2Y1RC9uUEFXbDA1YlBwLzkxamtDQmhxTE1nN2tPTApGcVVKdDhEOWlNNTJoa0F5UUZ3cVgwSk5LeTBTV0JNcnQydjZvSG9wblRPckwyMkczdGNVUlhVNlhnaE1DcUZPCmtiZFNBUUtCZ1FESmZ2djQrVHowTVZnOHdnMFpDdThFT0phaGFabFg5ck11dVRWcHlObm84Qi9VODdVU0VHVjAKZG9vc1YzSWxjQ1d3eEZQUkE3L3ZjbzlYM3B0VzVzQ1ErRnBmdE4yZkV1a051aTJTMS9RZEVGczNiYjNhd1NkdQo3aDBRaFBYeVN1RlQ1SncyVm94dnZubHZOTHIrYXVCM0hDVkVIb3pKMXZRUEtKV1J1NTFnRHc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
- name: kubernetes-admin2
user:
client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM4akNDQWRxZ0F3SUJBZ0lJUk5DL0tNVmhQT0F3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TURBNE1EVXdOVEV6TXpKYUZ3MHlNVEE0TURVd05URXpNelJhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXhUQmxUdjNMV3dacEp3UlQKVjg2eUsycUpmS1NnWXFYcXh3VlZOT3hUN3Ayb0VsRFlIZzZwWjdPNTltQ3grSDZFa01KbUVsamR1cFZXaFhDKwpNWDJHSEhRdk9KSEc1ck9lN3dTNTNUdEpxeXdZUUc2SGlrUjQ5WWtOODBBWjJEbHI2bVNhdTRTMkdpOXhKS0U1Cm8rbi9zMjBYYUpGeG9VOTU3MExVb0NQL2kzK0c4aFVFNk1aMG44eHRBYkJud2tlc1MwdXlIOTJsYzBOUjRabTgKN2ovempIWE9VTFZLa2FZM0Z0Tk5pdWRkVG1yOG5qZitlakdyeHFPOTRZMCtGU05hSHhhUW1yTUQwVERBYk8xaQpnV0o1UU1CQ2liNnI0dWFGOVplelZ2a01Yc1dWSUl2TUJOL29aRWYrMG4wazc5Sk9hRXVJMlpDOWcyc3ppbkVBCmp4azhud0lEQVFBQm95Y3dKVEFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFNRjNDT0lUOGFmSkhwd1I0L21KTk1qaGR3RFlOTWRrTi9RVgoxK2sxYnhOU0tiQ2JhMnVBN3RUUXhZcmYzb29MZzlBNkt3Z2UvSzJRcFZnYUZ5SDM4d0x3QVd2NEZjNklaOGwvClFZWXpZVnhERitXRW5uQ09USDVqVUFiaTg3aDlYNTNIbWcrR1QyVkZkTW95N3VVTnplcnYzVGo1cmpYUC9ENmIKSlg1UWV4T1A2Y3FZRjR5STlLelBBUjI1L3JSMktJZFNZVElad290TTQ4M3k5TzBhNlUxclhHWGJneUNoRFZHQwppcC9MQU8zWHRlSVl4b3FyVFBEUS9TUVdhRmszZGNoVGUycUNObGVvTFIrQlFjVHhVOFovc3Fuc3ZiaWUrZ3pVCmVsc3pYUjJselcvU3B2M05zL2t6czQ1UTZNT2JSOVNKMm1uQTFJUzJjV0dsQnd2bEFxRT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBeFRCbFR2M0xXd1pwSndSVFY4NnlLMnFKZktTZ1lxWHF4d1ZWTk94VDdwMm9FbERZCkhnNnBaN081OW1DeCtINkVrTUptRWxqZHVwVldoWEMrTVgyR0hIUXZPSkhHNXJPZTd3UzUzVHRKcXl3WVFHNkgKaWtSNDlZa044MEFaMkRscjZtU2F1NFMyR2k5eEpLRTVvK24vczIwWGFKRnhvVTk1NzBMVW9DUC9pMytHOGhVRQo2TVowbjh4dEFiQm53a2VzUzB1eUg5MmxjME5SNFptODdqL3pqSFhPVUxWS2thWTNGdE5OaXVkZFRtcjhuamYrCmVqR3J4cU85NFkwK0ZTTmFIeGFRbXJNRDBUREFiTzFpZ1dKNVFNQkNpYjZyNHVhRjlaZXpWdmtNWHNXVklJdk0KQk4vb1pFZiswbjBrNzlKT2FFdUkyWkM5ZzJzemluRUFqeGs4bndJREFRQUJBb0lCQUdlRXVacVhaL3Q0U1JtZgpqclZKSUlYa1h4c2RQY21yK3EzU29lR3ZzM3hRTGR5M3JrMVJPeVBpMlZ4N2ROTFVjSG1pWkgxN1E2UGhKRlBYClZUR2NDUnpFN2NsUDFsQmt1YW9YSEJBRlFPNW9ycTZjbC9GQmN4TExpWlJLOFladTFyVnRiVUhHTU9kY1FwU04KUXpwRzdPTGdBV3IyOStWTVdPSC9mcW8zVzc5NkhLcFE3UXpNY21UMmh0OFd0aU1xd2hzOWVsYkNsMk5HQjlqTQpyYllYNFZaeHVZazJQTUd3dm0xWE9HYjUvTHdGRXVneXh2UDJXakh4ckhkTnJ1bE95NS9ZbXVLWEFKeC9La2w5CjFnU1dOU2hySk84emZ3UTVqOGNERHd2WG5pT0EzUHZmcXNyT2pKSFc0dVVSeDhZaUhlOE1iVGorMCs2THRBTE4KbGRMdjJDRUNnWUVBL3hoZ2V2M1JYZnZSN3haS3hLUW1iRjFnSlB4bXJOYXkzaEY3WnRhNmZRMU5SS21McHVvQwpCWVhBd3FyV050ckRDTVN1OGtlamZqRHZUaDVsL2ZTREdwdFR1bVlMbFR0aFp1R2NielhBei9rMGN0K2QyczR6ClF1c1V3eW5OdTNjdG5tMVBuWUJWL1UySXpYcTZiVXFaRzREYi9HdzJQdnFtVStFK3o2YmdYazhDZ1lFQXhlTncKMk9WeWo1UjFKWnVYSDhZaTBWUmo0d0Ric2lRV1pvQ3ZMTGtNZnJPVUF6RHVMRDBDNUR2cVU3VTJjQ2NUdGQrYgovNGtEMUpzU0UyS2RsR0VqdzY0em1vOFhFSUg1ZVVCOXBwbWxlUXN2TGFWL0w3NlpRNGNqWjlVVWovRXBsV0pYCmJxMFE3YTdMdDdHZ0ZPcU15NldwL0lUMGpIZlNXRVBHR0J4TmVMRUNnWUJZZlU1S20rMmEycFh5aUJzYnA0MmsKbmFlU2JQUDVGT2tyVzRraEhXL0huV3VtVytJVDNxMGcyVXNLdkZSZTV2Z3MvSDFya2lwR3hoYmRWSEl3N0RibQpiZWRYaTdxTTQvSmZ5L2VHSkhGa002QmNFUElqU3RCQmNHYmRaSXozY05va1VPc0RTZGVLYlBEOXhvc1JDb1dlCklzOHMreVMrS0JpMmxEWkR0akd5eFFLQmdCUk5sWkpyV0NtckRhbmlJTXM5eE95OVA5SldrRFBpeC9tVVVtVEEKUjZ2b09OTnEvY1RFTGJ2RStmSEVzZVVuM1RuWHkyV1laVEEzQ00rOFduYVMwTnlPenZTWW9jRXIxTERBRVNRNgp0Mk9INWp0RXM1dnVia01qWmxxTTNpTTJseEdnbVYvZG1kTUpFQlcwdjd2bHBMUndOdFNWdjVGNTZiYnNuVEowCkZPN0JBb0dCQU81M2Z6RE00WkNVTkp3YTYvTVNlTXE1aEVEVWprWVBIaEIreEZnZFR6blpCU1dUN0RvZUpEWGcKWS9UMlRnSG40WTU0STBiRmNwUzZFQmI3NkFQTmhWOHZDaFJPZTRLRzkrTWlZb2s0dnZaVzl0R0xlaTdOdGlsYwpuMHAxb3V3eUduRytOVWljMm5KRXFWNTlMSVlONW9acFhySHc2S2R1eXFwbURWZnlxSzhTCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==
[root@client00 ~]#
好了,需要注意的是之前初始化模版配置里面users下面的client-certificate和client-key。需要自己加上-data后缀。
测试获取内容:
[root@client00 ~]# kubectl config use-context cluster1
Switched to context "cluster1".
[root@client00 ~]# kubectl config get-contexts
CURRENT NAME CLUSTER AUTHINFO NAMESPACE
* cluster1 cluster1 kubernetes-admin1
cluster2 cluster2 kubernetes-admin2
以上是准备好了两个集群内容。
[root@client00 ~]# helm init --upgrade -i registry.cn-shanghai.aliyuncs.com/gcrio_images/tiller:v2.16.9 --stable-repo-url https://kubernetes.oss-cn-shanghai.aliyuncs.com/charts --service-account tiller
[root@client00 ~]# helm version
Client: &version.Version{SemVer:"v2.16.9", GitCommit:"8ad7037828e5a0fca1009dabe290130da6368e39", GitTreeState:"clean"}
Server: &version.Version{SemVer:"v2.16.3", GitCommit:"1ee0254c86d4ed6887327dabed7aa7da29d7eb0d", GitTreeState:"clean"}
[root@client00 ~]# helm repo add kubefed-charts https://raw.githubusercontent.com/kubernetes-sigs/kubefed/master/charts
[root@client00 ~]# helm repo list
NAME URL
kubefed-charts https://raw.githubusercontent.com/kubernetes-sigs/kubefed/master/charts
[root@client00 ~]# helm search kubefed --devel
NAME CHART VERSION APP VERSION DESCRIPTION
kubefed-charts/kubefed 0.3.1 KubeFed helm chart
kubefed-charts/federation-v2 0.0.10 Kubernetes Federation V2 helm chart
[root@client00 ~]# helm install kubefed-charts/kubefed --name kubefed --version=0.3.1 --namespace kube-federation-system --devel --set controllermanager.repository=registry.cn-shanghai.aliyuncs.com/quayio_containers
kubectl -n kube-federation-system delete FederatedTypeConfig --all
kubectl delete crd $(kubectl get crd | grep -E 'kubefed.io' | awk '{print $1}')
helm delete --purge kubefed
kubefedctl join cluster1 --cluster-context cluster1 \
--host-cluster-context cluster1 --v=2
kubefedctl join cluster2 --cluster-context cluster2 \
--host-cluster-context cluster1 --v=2
[root@client00 ~]# kubectl -n kube-federation-system get kubefedclusters
NAME AGE READY
cluster1 20h True
cluster2 20h True
kubefedctl unjoin cluster2 --cluster-context cluster2 --host-cluster-context cluster1 --v=2
kubefedctl enable FederatedNamespace
kubefedctl enable FederatedDeployment
如下内容保存为test.yaml
apiVersion: types.kubefed.io/v1beta1
kind: FederatedNamespace
metadata:
name: test-namespace
namespace: test-namespace
spec:
placement:
clusters:
- name: cluster2
- name: cluster1
---
apiVersion: types.kubefed.io/v1beta1
kind: FederatedDeployment
metadata:
name: test-deployment
namespace: test-namespace
spec:
template:
metadata:
labels:
app: nginx
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- image: nginx
name: nginx
placement:
clusters:
- name: cluster2
- name: cluster1
执行文件
kubectl apply -f test.yaml
查看两边cluster是否部署成功
[root@client00 ~]# kubectl get pods --all-namespaces -o wide --context cluster1|grep test-deployment
test-namespace test-deployment-8cb559794-56g65 1/1 Running 0 16h 10.244.7.3 worker00
test-namespace test-deployment-8cb559794-v9dc6 1/1 Running 0 16h 10.244.7.2 worker00
test-namespace test-deployment-8cb559794-zfdpq 1/1 Running 0 16h 10.244.7.4 worker00
[root@client00 ~]# kubectl get pods --all-namespaces -o wide --context cluster2|grep test-deployment
test-namespace test-deployment-8cb559794-4z6hx 1/1 Running 0 16h 10.244.4.6 worker01
test-namespace test-deployment-8cb559794-rrjj7 1/1 Running 0 16h 10.244.5.8 worker02
test-namespace test-deployment-8cb559794-xt695 1/1 Running 0 16h 10.244.5.7 worker02
好了,能看到以上结果说明两个集群联邦成功了。想了解关于更多联邦详情,请仔细阅读官方文档。