云原生 | k8s安装KubeSphere
前置条件
准备已经安装好的k8s集群，最少一个master节点和工作节点,master节点已经初始化，工作节点已经加入到master节点。
当前版本资源：
k8s版本：1.21.1
KubeSphere版本：v3.1.1
192.168.112.178 k8s-master-1
192.168.112.179 k8s-master-2
192.168.112.180 k8s-master-3
192.168.112.181 k8s-node-1
192.168.112.182 k8s-node-2
192.168.112.183 k8s-node-3
192.168.112.184 k8s-node-4
192.168.112.185 k8s-node-5
192.168.112.186 k8s-node-6
192.168.112.187 k8s-node-7
192.168.112.188 k8s-node-8
192.168.112.189 k8s-node-9
192.168.112.190 k8s-node-10
nfs服务器：192.168.112.177
一.配置k8s集群中的默认存储类型NFS
1.所有master，node节点，nfs节点执行
# 所有机器安装
yum install -y nfs-utils rpcbind
# 所有机器设置开机自启 & 现在启动 -- 远程绑定服务
systemctl enable rpcbind --now
# nfs节点（192.168.112.177）
systemctl enable nfs-server --now2.nfs节点（192.168.112.177）执行
# nfs节点
测试环境：
echo "/nfs/data/ *(insecure,rw,sync,no_root_squash)" > /etc/exports
生产环境：
"/nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash) /nfs 192.168.123.81/24(rw,async,no_root_squash)" > /etc/exports
# 新建共享目录
mkdir -p /nfs
chmod -R 777 /nfs
# 配置生效
exportfs -r
# 查看
exportfs3.各主从节点配置
192.168.112.177替换成你的各master，node节点ip
# 查看远程机器有哪些目录可以同步 -- 使用nfs机器ip地址
# 执行以下命令挂载 nfs 服务器上的共享目录到本机路径
showmount -e 192.168.112.177
可以看到在192.168.112.177的主机上共享了/nfs-data 目录
# 各个主从节点新建/nfs/data目录
mkdir -p /nfs/data
chmod -R 777 /nfs/data
# 同步远程机器数据，各主从节点上执行以下命令
mount -t nfs 192.168.112.177:/nfs/data /nfs/data4.各主从节点开启fstab自动挂载设置
# nfs节点192.168.112.177开启服务自启动即可
# 各主从节点
vim /etc/fstab
192.168.112.177:/nfs /nfs/nfs-data/ nfs4 defaults 0 0
5.测试
# 在任意机器写入一个测试文件
echo "hello world" > /nfs/data/test.txt
# 在其它机器查看数据
cat /nfs/data/test.txt二.配置动态供应的默认存储类
1.创建名为storageclass.yaml的文件
内容如下：需要修改
将192.168.177.132替换成你的master节点地址，其余不变
## 创建了一个存储类
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: nfs-storage
annotations:
storageclass.kubernetes.io/is-default-class: "true" #---设置为默认的storageclass
provisioner: nfs-client #---动态卷分配者名称，必须和上面创建的"provisioner"变量中设置的Name一致
parameters:
archiveOnDelete: "true" #---设置为"false"时删除PVC不会保留数据,"true"则保留数据
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nfs-client-provisioner
labels:
app: nfs-client-provisioner
namespace: default #与RBAC文件中的namespace保持一致
spec:
replicas: 1
selector:
matchLabels:
app: nfs-client-provisioner
strategy:
type: Recreate
selector:
matchLabels:
app: nfs-client-provisioner
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
image: quay.io/external_storage/nfs-client-provisioner:latest
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: nfs-client #provisioner名称,请确保该名称与 nfs-StorageClass.yaml文件中的provisioner名称保持一致
- name: NFS_SERVER
value: 192.168.112.177 #NFS Server IP地址
- name: NFS_PATH
value: /nfs/data #NFS挂载卷
volumes:
- name: nfs-client-root
nfs:
server: 192.168.112.177 #NFS Server IP地址
path: /nfs/data #NFS 挂载卷
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
# replace with namespace where provisioner is deployed
namespace: default
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
namespace: default
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: default
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: default
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io2.执行storageclass.yaml
kubectl apply -f storageclass.yaml3.查看存储类
[root@k8s-master-1 nfs.4.0]# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-storage (default) nfs-client Delete Immediate false 18h
[root@k8s-master-1 nfs.4.0]# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
default nfs-client-provisioner-688fcbdd54-4sl6h 1/1 Running 98 18h4.创建一个PVC测试一下动态供应能力
1.创建名为pmm-v.yaml的文件，内容如下：不用修改
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: pmm-v
namespace: default #---指定namespace为monitoring
spec:
storageClassName: nfs-storage #---需要与上面创建的storageclass的名称一致
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi2.执行pvc.yaml
kubectl apply -f pmm-v.yaml3.查看pvc,pv
kubectl get pvc
kubectl get pv4.出现pvc处于pending状态
[root@k8s-master-1 nfs]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pmm-data Pending nfs-storage 16s
pmm-v Pending nfs-storage 129m如果K8s版本为1.20.x
在三个master节点/etc/kubernetes/manifests/kube-apiserver.yaml的command中添加：
- --feature-gates=RemoveSelfLink=false然后重新生成kube-apiserver.yaml否则会报错：
Kubernetes v1.20.13 报"unexpected error getting claim reference: selfLink was empty, can’t make reference"下载yaml文件
[root@k8s-master-1 nfs]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
pmm-data Bound pvc-3f6378d3-6c00-4fc4-9a71-3d68351dd44d 50Gi RWO nfs-storage 15h
pmm-v Bound pvc-15948d50-d101-4a5e-bd54-74d700b91faf 10Gi RWO nfs-storage 17h
[root@k8s-master-1 nfs]# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-15948d50-d101-4a5e-bd54-74d700b91faf 10Gi RWO Delete Bound default/pmm-v nfs-storage 15h
pvc-3f6378d3-6c00-4fc4-9a71-3d68351dd44d 50Gi RWO Delete Bound default/pmm-data nfs-storage 15h
三.安装群指标监控组件metrics-server
集群指标监控组件
创建名为metrics-server.yaml的文件
内容如下：不用修改
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
rbac.authorization.k8s.io/aggregate-to-admin: "true"
rbac.authorization.k8s.io/aggregate-to-edit: "true"
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: system:aggregated-metrics-reader
rules:
- apiGroups:
- metrics.k8s.io
resources:
- pods
- nodes
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- nodes/stats
- namespaces
- configmaps
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server-auth-reader
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server:system:auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
ports:
- name: https
port: 443
protocol: TCP
targetPort: https
selector:
k8s-app: metrics-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: metrics-server
strategy:
rollingUpdate:
maxUnavailable: 0
template:
metadata:
labels:
k8s-app: metrics-server
spec:
containers:
- args:
- --cert-dir=/tmp
- --kubelet-insecure-tls
- --secure-port=4443
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --kubelet-use-node-status-port
image: registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images/metrics-server:v0.4.3
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: livez
port: https
scheme: HTTPS
periodSeconds: 10
name: metrics-server
ports:
- containerPort: 4443
name: https
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: readyz
port: https
scheme: HTTPS
periodSeconds: 10
securityContext:
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
volumeMounts:
- mountPath: tmp
name: tmp-dir
nodeSelector:
kubernetes.io/os: linux
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
volumes:
- emptyDir: {}
name: tmp-dir
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
labels:
k8s-app: metrics-server
name: v1beta1.metrics.k8s.io
spec:
group: metrics.k8s.io
groupPriorityMinimum: 100
insecureSkipTLSVerify: true
service:
name: metrics-server
namespace: kube-system
version: v1beta1
versionPriority: 1001.修改kube-apiserver.yaml
修改每个 API Server 的 kube-apiserver.yaml 配置开启 Aggregator Routing
不开启Aggregator Routing的话不能使用
vi /etc/kubernetes/manifests/kube-apiserver.yaml添加--enable-aggregator-routing=true到yaml文件
spec:
containers:
- command:
- kube-apiserver
- --advertise-address=192.168.177.131
- --allow-privileged=true
- --authorization-mode=Node,RBAC
- --client-ca-file=/etc/kubernetes/pki/ca.crt
- --enable-admission-plugins=NodeRestriction
- --enable-bootstrap-token-auth=true
- --enable-aggregator-routing=true #添加，开启Aggregator Routing（聚合路由）2.重启kubelet
systemctl daemon-reload
systemctl restart kubelet3.拉取镜像
docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.4.34.执行metrics-server.yaml
kubectl apply -f metrics-server.yaml5.查看Metrics Server服务状态
kubectl get pods -n kube-system
metrics-server-7457bfc9f4-mj6zh 1/1 Running 0 109s6.查看数据
需要先配置在/etc/kubernetes/manifests/kube-apiserver.yaml添加以下才能使用以下命令
- --enable-aggregator-routing=true
kubectl top nodes
kubectl top pods
kubectl top pods -A四.安装KubeSphere
1.下载kubesphere-installer.yaml和cluster-configuration.yaml
wget https://github.com/kubesphere/ks-installer/releases/download/v3.1.1/kubesphere-installer.yaml
wget https://github.com/kubesphere/ks-installer/releases/download/v3.1.1/cluster-configuration.yaml2.安装
修改配置文件
vim cluster-configuration.yaml vim cluster-configuration.yaml
[root@master ~]# cat cluster-configuration.yaml
---
apiVersion: installer.kubesphere.io/v1alpha1
kind: ClusterConfiguration
metadata:
name: ks-installer
namespace: kubesphere-system
labels:
version: v3.1.1
spec:
persistence:
storageClass: "" #这里保持默认即可，因为偶们有了默认的存储类
authentication:
jwtSecret: "" # Keep the jwtSecret consistent with the Host Cluster. Retrieve the jwtSecret by executing "kubectl -n kubesphere-system get cm kubesphere-config -o yaml | grep -v "apiVersion" | grep jwtSecret" on the Host Cluster.
local_registry: "" # Add your private registry address if it is needed.
etcd:
monitoring: true # 改为"true"，表示开启etcd的监控功能
endpointIps: 192.168.10.27 # 改为自己的master节点IP地址
port: 2379 # etcd port.
tlsEnable: true
common:
redis:
enabled: true #改为"true"，开启redis功能
openldap:
enabled: true #改为"true"，开启轻量级目录协议
minioVolumeSize: 20Gi # Minio PVC size.
openldapVolumeSize: 2Gi # openldap PVC size.
redisVolumSize: 2Gi # Redis PVC size.
monitoring:
# type: external # Whether to specify the external prometheus stack, and need to modify the endpoint at the next line.
endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 # Prometheus endpoint to get metrics data.
es: # Storage backend for logging, events and auditing.
# elasticsearchMasterReplicas: 1 # The total number of master nodes. Even numbers are not allowed.
# elasticsearchDataReplicas: 1 # The total number of data nodes.
elasticsearchMasterVolumeSize: 4Gi # The volume size of Elasticsearch master nodes.
elasticsearchDataVolumeSize: 20Gi # The volume size of Elasticsearch data nodes.
logMaxAge: 7 # Log retention time in built-in Elasticsearch. It is 7 days by default.
elkPrefix: logstash # The string making up index names. The index name will be formatted as ks--log.
basicAuth:
enabled: false #此处的"false"不用改为"true",这个标识在开启监控功能之后是否要连接ElasticSearch的账户和密码，此处不用
username: ""
password: ""
externalElasticsearchUrl: ""
externalElasticsearchPort: ""
console:
enableMultiLogin: true # Enable or disable simultaneous logins. It allows different users to log in with the same account at the same time.
port: 30880
alerting: # (CPU: 0.1 Core, Memory: 100 MiB) It enables users to customize alerting policies to send messages to receivers in time with different time intervals and alerting levels to choose from.
enabled: true # 改为"true"，开启告警功能
# thanosruler:
# replicas: 1
# resources: {}
auditing:
enabled: true # 改为"true"，开启审计功能
devops: # (CPU: 0.47 Core, Memory: 8.6 G) Provide an out-of-the-box CI/CD system based on Jenkins, and automated workflow tools including Source-to-Image & Binary-to-Image.
enabled: true # 改为"true"，开启DevOps功能
jenkinsMemoryLim: 2Gi # Jenkins memory limit.
jenkinsMemoryReq: 1500Mi # Jenkins memory request.
jenkinsVolumeSize: 8Gi # Jenkins volume size.
jenkinsJavaOpts_Xms: 512m # The following three fields are JVM parameters.
jenkinsJavaOpts_Xmx: 512m
jenkinsJavaOpts_MaxRAM: 2g
events: # Provide a graphical web console for Kubernetes Events exporting, filtering and alerting in multi-tenant Kubernetes clusters.
enabled: true # 改为"true"，开启集群的事件功能
ruler:
enabled: true
replicas: 2
logging: # (CPU: 57 m, Memory: 2.76 G) Flexible logging functions are provided for log query, collection and management in a unified console. Additional log collectors can be added, such as Elasticsearch, Kafka and Fluentd.
enabled: true # 改为"true"，开启日志功能
logsidecar:
enabled: true
replicas: 2
metrics_server: # (CPU: 56 m, Memory: 44.35 MiB) It enables HPA (Horizontal Pod Autoscaler).
enabled: false # 这个不用修改，因为在上卖弄我们已经安装过了，如果这里开启，镜像是官方的，会拉取镜像失败
monitoring:
storageClass: ""
# prometheusReplicas: 1 # Prometheus replicas are responsible for monitoring different segments of data source and providing high availability.
prometheusMemoryRequest: 400Mi # Prometheus request memory.
prometheusVolumeSize: 20Gi # Prometheus PVC size.
# alertmanagerReplicas: 1 # AlertManager Replicas.
multicluster:
clusterRole: none # host | member | none # You can install a solo cluster, or specify it as the Host or Member Cluster.
network:
networkpolicy: # Network policies allow network isolation within the same cluster, which means firewalls can be set up between certain instances (Pods).
# Make sure that the CNI network plugin used by the cluster supports NetworkPolicy. There are a number of CNI network plugins that support NetworkPolicy, including Calico, Cilium, Kube-router, Romana and Weave Net.
enabled: true # 改为"true"，开启网络策略
ippool: # Use Pod IP Pools to manage the Pod network address space. Pods to be created can be assigned IP addresses from a Pod IP Pool.
type: none #如果你的网络插件是calico，需要修改为"calico"，这里我是Flannel，保持默认。
topology: # Use Service Topology to view Service-to-Service communication based on Weave Scope.
type: none # Specify "weave-scope" for this field to enable Service Topology. "none" means that Service Topology is disabled.
openpitrix: # An App Store that is accessible to all platform tenants. You can use it to manage apps across their entire lifecycle.
store:
enabled: true # 改为"true"，开启应用商店
servicemesh: # (0.3 Core, 300 MiB) Provide fine-grained traffic management, observability and tracing, and visualized traffic topology.
enabled: true # 改为"true"，开启微服务治理
kubeedge: # Add edge nodes to your cluster and deploy workloads on edge nodes.
enabled: false # 这个就不修改了，这个是边缘服务，我们也没有边缘的设备。
cloudCore:
nodeSelector: {"node-role.kubernetes.io/worker": ""}
tolerations: []
cloudhubPort: "10000"
cloudhubQuicPort: "10001"
cloudhubHttpsPort: "10002"
cloudstreamPort: "10003"
tunnelPort: "10004"
cloudHub:
advertiseAddress: # At least a public IP address or an IP address which can be accessed by edge nodes must be provided.
- "" # Note that once KubeEdge is enabled, CloudCore will malfunction if the address is not provided.
nodeLimit: "100"
service:
cloudhubNodePort: "30000"
cloudhubQuicNodePort: "30001"
cloudhubHttpsNodePort: "30002"
cloudstreamNodePort: "30003"
tunnelNodePort: "30004"
edgeWatcher:
nodeSelector: {"node-role.kubernetes.io/worker": ""}
tolerations: []
edgeWatcherAgent:
nodeSelector: {"node-role.kubernetes.io/worker": ""}
tolerations: [] kubectl apply -f kubesphere-installer.yaml
kubectl apply -f cluster-configuration.yaml1.检查安装日志
执行以下命令查看日志，需要点时间加载，耐心等待
kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l app=ks-install -o jsonpath='{.items[0].metadata.name}') -f界面出现以下内容，说明已经安装成功
2.查看所有 Pod 是否在 KubeSphere 的相关命名空间中正常运行
kubectl get pod --all-namespaces3.检查控制台的端口
[root@k8s-master-1 k8s]# kubectl get svc/ks-console -n kubesphere-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ks-console NodePort 10.103.27.101 80:30880/TCP 11m 4.浏览器访问集群任意机器IP:30880
192.168.112.177:30880
192.168.112.178:30880
192.168.112.180:30880初始账号密码: admin/P@88w0rd
登录截图
五.卸载KubeSphere
wget https://raw.githubusercontent.com/kubesphere/ks-installer/release-3.1/scripts/kubesphere-delete.sh && sh kubesphere-delete.sh