K8S cluster with multi-masters on Azure VM
拓扑参考：
在 Azure VM 实例上部署 KubeSphere
-
基础模板
需要修改 IP 地址和 VM Image的可以在模板中修改。{ "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { "vmNamePrefix": { "defaultValue": "master-", "type": "String", "metadata": { "description": "The name of your VM master node." } }, "vmssName": { "defaultValue": "node", "type": "String", "metadata": { "description": "The name of your VMSS cluster." } }, "location": { "defaultValue": "[resourceGroup().location]", "type": "String", "metadata": { "description": "Location for all resources." } }, "adminUsername": { "type": "String", "metadata": { "description": "Username for the Virtual Machine." } }, "adminKey": { "type": "SecureString", "metadata": { "description": "SSH Key for the Virtual Machine." } }, "defaultMasterCount": { "defaultValue": 3, "type": "Int", "metadata": { "description": "The default instances count of master" } }, "defaultNodeCount": { "defaultValue": 3, "type": "Int", "metadata": { "description": "The initial node size of your VMSS cluster." } }, "minNodeCount": { "defaultValue": 1, "type": "Int", "metadata": { "description": "The min node size of your VMSS cluster." } }, "maxNodeCount": { "defaultValue": 20, "type": "Int", "metadata": { "description": "The max node size of your VMSS cluster." } }, "dnsLabelPrefix": { "defaultValue": "[toLower(concat('k8s-cluster-', uniqueString(resourceGroup().id)))]", "type": "String", "metadata": { "description": "Unique DNS Name for the Public IP used to access the Virtual Machine." } }, "vmSize": { "defaultValue": "Standard_DS2_v2", "type": "String", "metadata": { "description": "The size of the VM" } }, "virtualNetworkName": { "defaultValue": "vNetwork", "type": "String", "metadata": { "description": "Name of the Virtual Network" } }, "subnetName": { "defaultValue": "Subnet", "type": "String", "metadata": { "description": "Name of the subnet in the virtual network" } }, "vmssSubnetName": { "defaultValue": "nodeSubnet", "type": "String", "metadata": { "description": "Name of the VMSS subnet in the virtual network" } }, "publicLBName": { "defaultValue": "publicLB", "type": "String", "metadata": { "description": "Internal Load Balancer name" } } }, "variables": { "publicIPAddressName": "[concat(parameters('publicLBName'), 'IP' )]", "availabilitySetName": "masterAvSet", "networkInterfaceName": "[concat(parameters('vmNamePrefix'),'Interface')]", "networkSecurityGroupName": "[concat(parameters('virtualNetworkName'),'nsg-default')]", "subnetRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('virtualNetworkName'), parameters('subnetName'))]", "vmssSubnetRef": "[resourceId('Microsoft.Network/virtualNetworks/subnets', parameters('virtualNetworkName'), parameters('vmssSubnetName'))]", "osDiskType": "Standard_LRS", "publicLBID": "[resourceId('Microsoft.Network/loadBalancers',parameters('publicLBName'))]" }, "resources": [ { "type": "Microsoft.Network/networkSecurityGroups", "apiVersion": "2015-06-15", "name": "[variables('networkSecurityGroupName')]", "location": "[parameters('location')]", "properties": { "securityRules": [ { "name": "Port_SSH", "properties": { "description": "SSH", "protocol": "*", "sourcePortRange": "*", "destinationPortRange": "22", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 100, "direction": "Inbound", "sourcePortRanges": [], "destinationPortRanges": [], "sourceAddressPrefixes": [], "destinationAddressPrefixes": [] } }, { "name": "Port_API_Server", "properties": { "description": "k8s API Server", "protocol": "TCP", "sourcePortRange": "*", "destinationPortRange": "6443", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 140, "direction": "Inbound", "sourcePortRanges": [], "destinationPortRanges": [], "sourceAddressPrefixes": [], "destinationAddressPrefixes": [] } }, { "name": "Port_Http", "properties": { "description": "Web", "protocol": "TCP", "sourcePortRange": "*", "destinationPortRange": "80", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 120, "direction": "Inbound", "sourcePortRanges": [], "destinationPortRanges": [], "sourceAddressPrefixes": [], "destinationAddressPrefixes": [] } }, { "name": "Port_Https", "properties": { "protocol": "TCP", "sourcePortRange": "*", "destinationPortRange": "443", "sourceAddressPrefix": "*", "destinationAddressPrefix": "*", "access": "Allow", "priority": 130, "direction": "Inbound", "sourcePortRanges": [], "destinationPortRanges": [], "sourceAddressPrefixes": [], "destinationAddressPrefixes": [] } } ] } }, { "type": "Microsoft.Network/virtualNetworks", "apiVersion": "2019-11-01", "name": "[parameters('virtualNetworkName')]", "location": "[parameters('location')]", "dependsOn": [ "[resourceId('Microsoft.Network/networkSecurityGroups/', variables('networkSecurityGroupName'))]" ], "properties": { "addressSpace": { "addressPrefixes": [ "10.211.0.0/16" ] }, "subnets": [ { "name": "[parameters('vmssSubnetName')]", "properties": { "networkSecurityGroup": { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]" }, "addressPrefix": "10.211.0.0/24" } }, { "name": "[parameters('subnetName')]", "properties": { "networkSecurityGroup": { "id": "[resourceId('Microsoft.Network/networkSecurityGroups', variables('networkSecurityGroupName'))]" }, "addressPrefix": "10.211.1.0/24" } } ] } }, { "type": "Microsoft.Network/networkInterfaces", "apiVersion": "2019-11-01", "name": "[concat(variables('networkInterfaceName'), copyindex())]", "location": "[parameters('location')]", "dependsOn": [ "[resourceId('Microsoft.Network/virtualNetworks/', parameters('virtualNetworkName'))]", "[resourceId('Microsoft.Network/loadBalancers/', parameters('publicLBName'))]" ], "properties": { "ipConfigurations": [ { "name": "ipconfig1", "properties": { "subnet": { "id": "[variables('subnetRef')]" }, "loadBalancerBackendAddressPools": [ { "id": "[concat(variables('publicLBID'), '/backendAddressPools/BackendPoolMaster')]" } ], "loadBalancerInboundNatRules": [ { "id": "[concat(variables('publicLBID'), '/inboundNatRules/lbNAT-master',copyindex())]" } ], "privateIPAllocationMethod": "Dynamic" } } ] }, "copy": { "name": "nicLoop", "count": "[parameters('defaultMasterCount')]" } }, { "type": "Microsoft.Network/publicIPAddresses", "apiVersion": "2019-09-01", "name": "[variables('publicIPAddressName')]", "location": "[parameters('location')]", "sku": { "name": "Standard" }, "properties": { "publicIPAllocationMethod": "Static", "publicIPAddressVersion": "IPv4", "dnsSettings": { "domainNameLabel": "[parameters('dnsLabelPrefix')]" }, "idleTimeoutInMinutes": 10 } }, { "type": "Microsoft.Network/loadBalancers", "apiVersion": "2018-06-01", "name": "[parameters('publicLBName')]", "location": "[parameters('location')]", "dependsOn": [ "[concat('Microsoft.Network/publicIPAddresses/', variables('publicIPAddressName'))]" ], "sku": { "name": "Standard" }, "properties": { "frontendIPConfigurations": [ { "name": "LoadBalancerFrontEnd", "properties": { "publicIPAddress": { "id": "[resourceId('Microsoft.Network/publicIPAddresses',variables('publicIPAddressName'))]" } } } ], "backendAddressPools": [ { "name": "BackendPoolNode" }, { "name": "BackendPoolMaster" } ], "loadBalancingRules": [ { "name": "HttpLBRule", "properties": { "frontendIPConfiguration": { "id": "[concat(variables('publicLBID'),'/frontendIPConfigurations/LoadBalancerFrontEnd')]" }, "backendAddressPool": { "id": "[concat(variables('publicLBID'),'/backendAddressPools/BackendPoolNode')]" }, "protocol": "Tcp", "frontendPort": 80, "backendPort": 80, "enableFloatingIP": false, "disableOutboundSnat": false, "idleTimeoutInMinutes": 5, "probe": { "id": "[concat(variables('publicLBID'),'/probes/tcpProbe')]" } } }, { "name": "APILBRule", "properties": { "frontendIPConfiguration": { "id": "[concat(variables('publicLBID'),'/frontendIPConfigurations/LoadBalancerFrontEnd')]" }, "backendAddressPool": { "id": "[concat(variables('publicLBID'),'/backendAddressPools/BackendPoolMaster')]" }, "protocol": "Tcp", "frontendPort": 6443, "backendPort": 6443, "enableFloatingIP": false, "disableOutboundSnat": false, "idleTimeoutInMinutes": 5, "probe": { "id": "[concat(variables('publicLBID'),'/probes/apitcpProbe')]" } } } ], "probes": [ { "name": "tcpProbe", "properties": { "protocol": "Tcp", "port": 80, "intervalInSeconds": 5, "numberOfProbes": 2 } }, { "name": "apitcpProbe", "properties": { "protocol": "Tcp", "port": 6443, "intervalInSeconds": 5, "numberOfProbes": 2 } } ], "inboundNatRules": [ { "name": "lbNAT-master0", "properties": { "frontendIPConfiguration": { "id": "[concat(variables('publicLBID'),'/frontendIPConfigurations/LoadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": "50200", "backendPort": "22" } }, { "name": "lbNAT-master1", "properties": { "frontendIPConfiguration": { "id": "[concat(variables('publicLBID'),'/frontendIPConfigurations/LoadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": "50201", "backendPort": "22" } }, { "name": "lbNAT-master2", "properties": { "frontendIPConfiguration": { "id": "[concat(variables('publicLBID'),'/frontendIPConfigurations/LoadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPort": "50202", "backendPort": "22" } } ], "inboundNatPools": [ { "name": "lbNAT-node", "properties": { "frontendIPConfiguration": { "id": "[concat(variables('publicLBID'),'/frontendIPConfigurations/LoadBalancerFrontEnd')]" }, "protocol": "Tcp", "frontendPortRangeStart": 50100, "frontendPortRangeEnd": 50199, "backendPort": 22 } } ] } }, { "type": "Microsoft.Compute/availabilitySets", "apiVersion": "2016-04-30-preview", "name": "[variables('availabilitySetName')]", "location": "[parameters('location')]", "properties": { "platformFaultDomainCount": 2, "platformUpdateDomainCount": 2, "managed": true } }, { "type": "Microsoft.Compute/virtualMachines", "apiVersion": "2019-07-01", "name": "[concat(parameters('vmNamePrefix'), copyindex())]", "location": "[parameters('location')]", "dependsOn": [ "[concat('Microsoft.Network/networkInterfaces/', variables('networkInterfaceName'), copyindex())]", "[concat('Microsoft.Compute/availabilitySets/', variables('availabilitySetName'))]", "[concat('Microsoft.Compute/virtualMachineScaleSets/', parameters('vmssName'))]" ], "properties": { "availabilitySet": { "id": "[resourceId('Microsoft.Compute/availabilitySets',variables('availabilitySetName'))]" }, "hardwareProfile": { "vmSize": "[parameters('vmSize')]" }, "storageProfile": { "osDisk": { "createOption": "FromImage", "managedDisk": { "storageAccountType": "[variables('osDiskType')]" } }, "imageReference": { "publisher": "Canonical", "offer": "0001-com-ubuntu-server-focal", "sku": "20_04-lts-gen2", "version": "latest" } }, "networkProfile": { "networkInterfaces": [ { "id": "[resourceId('Microsoft.Network/networkInterfaces',concat(variables('networkInterfaceName'),copyindex()))]" } ] }, "osProfile": { "computerName": "[concat(parameters('vmNamePrefix'), copyindex())]", "adminUsername": "[parameters('adminUsername')]", "adminPassword": "[parameters('adminKey')]", "linuxConfiguration": { "disablePasswordAuthentication": true, "ssh": { "publicKeys": [ { "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]", "keyData": "[parameters('adminKey')]" } ] } } } }, "copy": { "name": "virtualMachineLoop", "count": "[parameters('defaultMasterCount')]" } }, { "type": "Microsoft.Compute/virtualMachineScaleSets", "apiVersion": "2019-07-01", "name": "[parameters('vmssName')]", "location": "[parameters('location')]", "dependsOn": [ "[concat('Microsoft.Network/virtualNetworks/', parameters('virtualNetworkName'))]", "[concat('Microsoft.Network/loadBalancers/', parameters('publicLBName'))]" ], "tags": { "cluster-autoscaler-enabled": "true", "cluster-autoscaler-name": "[resourceGroup().name]", "min": "[parameters('minNodeCount')]", "max": "[parameters('maxNodeCount')]", "poolName": "[parameters('vmssName')]" }, "sku": { "name": "[parameters('vmSize')]", "tier": "Standard", "capacity": "[parameters('defaultNodeCount')]" }, "properties": { "overprovision": false, "upgradePolicy": { "mode": "Manual" }, "virtualMachineProfile": { "storageProfile": { "osDisk": { "createOption": "FromImage", "caching": "ReadWrite" }, "imageReference": { "publisher": "Canonical", "offer": "0001-com-ubuntu-server-focal", "sku": "20_04-lts-gen2", "version": "latest" } }, "osProfile": { "computerNamePrefix": "[parameters('vmssName')]", "adminUsername": "[parameters('adminUsername')]", "adminPassword": "[parameters('adminKey')]", "linuxConfiguration": { "disablePasswordAuthentication": true, "ssh": { "publicKeys": [ { "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]", "keyData": "[parameters('adminKey')]" } ] } } }, "networkProfile": { "networkInterfaceConfigurations": [ { "name": "[concat(parameters('vmssName'),'nic')]", "properties": { "primary": true, "ipConfigurations": [ { "name": "[concat('ipconfigVmss', parameters('vmssName'))]", "properties": { "subnet": { "id": "[variables('vmssSubnetRef')]" }, "loadBalancerBackendAddressPools": [ { "id": "[concat(variables('publicLBID'), '/backendAddressPools/BackendPoolNode')]" } ], "loadBalancerInboundNatPools": [ { "id": "[concat(variables('publicLBID'), '/inboundNatPools/lbNAT-node')]" } ] } } ] } } ] } } } } ], "outputs": { "adminUsername": { "type": "String", "value": "[parameters('adminUsername')]" }, "hostname": { "type": "String", "value": "[reference(variables('publicIPAddressName')).dnsSettings.fqdn]" }, "sshCommand": { "type": "String", "value": "[concat('ssh ', parameters('adminUsername'), '@', reference(variables('publicIPAddressName')).dnsSettings.fqdn)]" } } }-
可以修改 master 和 node 的名字前缀、部署区域、数量和 VM 类型
- ssh 22 对外由 LB 配置 NAT 端口实现，如配置文件中 50200 → master-0
-
已经包含的规则转换（不含 30880）
服务 协议 规则 后端端口 前端端口 节点池 ssh TCP 入站 NAT 22 50200, 50201, 50202, 50100~50199 主节点, 普通节点 api 服务器 TCP 负载均衡 6443 6443 主节点 ks 控制台 TCP 负载均衡 30880 30880 主节点 http TCP 负载均衡 80 80 普通节点 https TCP 负载均衡 443 443 普通节点
-
- node 使用 VMSS
- ssh 22 对外由 LB 配置 NAT 端口实现，如配置文件中 50200 → master-0
-
-
部署 K8S cluster
最简单的方式还是用 kk 完成，注意在 kubernetes 1.24 以后，psp 弃用，在 kk 中还有 psp 权限管理，安装的时候会报错。建议使用 1.23.10，也是 kk 现在默认的版本。-
证书传输 → Master-0
scp -i zyi.pem -P 50200 zyi.pem zyi@20.247.0.170:/home/zyi/.ssh/ -
每一台安装 socat 等必须的软件
ssh -i zyi.pem -p50200 zyi@20.247.0.170 'sudo apt install socat conntrack' -
登录到 Mater-0
ssh -i zyi.pem -p50200 zyi@20.247.0.170 -
下载 kk 并赋予可执行权限
curl -sfL https://get-kk.kubesphere.io | VERSION=v3.0.10 sh - chmod +x kk-
创建配置文件模板
./kk create config --with-kubesphere v3.3.2 --with-kubernetes v1.22.12-
KubeSphere 3.3 对应 Kubernetes 版本推荐：v1.20.x、v1.21.x、* v1.22.x、* v1.23.x 和 * v1.24.x。带星号的版本可能出现边缘节点部分功能不可用的情况。因此，如需使用边缘节点，推荐安装 v1.21.x。如果未指定 Kubernetes 版本，KubeKey 将默认安装 Kubernetes v1.23.10。有关支持的 Kubernetes 版本请参阅支持矩阵。
-
如果在此步骤中的命令中未添加标志
-with-kubesphere，则不会部署 KubeSphere，除非您使用配置文件中的addons字段进行安装，或稍后使用./kk create cluster时再次添加此标志。 -
如果在未指定 KubeSphere 版本的情况下添加标志 --with kubesphere`，将安装 KubeSphere 的最新版本。
-
修改的内容用红色标注
apiVersion: kubekey.kubesphere.io/v1alpha2 kind: Cluster metadata: name: kubeCluster spec: hosts: - {name: master-0, address: 20.210.0.156, port: 50200, internalAddress: 10.211.1.5, user: zyi, privateKeyPath: "~/.ssh/zyi.pem"} - {name: master-1, address: 20.210.0.156, port: 50201, internalAddress: 10.211.1.6, user: zyi, privateKeyPath: "~/.ssh/zyi.pem"} - {name: master-2, address: 20.210.0.156, port: 50202, internalAddress: 10.211.1.4, user: zyi, privateKeyPath: "~/.ssh/zyi.pem"} - {name: node000000, address: 20.210.0.156, port: 50100, internalAddress: 10.211.0.4, user: zyi, privateKeyPath: "~/.ssh/zyi.pem"} - {name: node000001, address: 20.210.0.156, port: 50101, internalAddress: 10.211.0.5, user: zyi, privateKeyPath: "~/.ssh/zyi.pem"} - {name: node000002, address: 20.210.0.156, port: 50102, internalAddress: 10.211.0.6, user: zyi, privateKeyPath: "~/.ssh/zyi.pem"} roleGroups: etcd: - master-0 - master-1 - master-2 control-plane: - master-0 - master-1 - master-2 worker: - node000000 - node000001 - node000002 controlPlaneEndpoint: ## Internal loadbalancer for apiservers # internalLoadbalancer: haproxy domain: lb.etaon.lab address: "20.210.0.156" port: 6443 kubernetes: version: v1.23.10 clusterName: cluster.local autoRenewCerts: true containerManager: docker etcd: type: kubekey network: plugin: flannel kubePodsCIDR: 10.233.64.0/18 kubeServiceCIDR: 10.233.0.0/18 ## multus support. https://github.com/k8snetworkplumbingwg/multus-cni multusCNI: enabled: false registry: privateRegistry: "" namespaceOverride: "" registryMirrors: [] insecureRegistries: [] addons: [] --- apiVersion: installer.kubesphere.io/v1alpha1 kind: ClusterConfiguration metadata: name: ks-installer namespace: kubesphere-system labels: version: v3.3.2 spec: persistence: storageClass: "" authentication: jwtSecret: "" zone: "" local_registry: "" namespace_override: "" # dev_tag: "" etcd: monitoring: false endpointIps: localhost port: 2379 tlsEnable: true common: core: console: enableMultiLogin: true port: 30880 type: NodePort # apiserver: # resources: {} # controllerManager: # resources: {} redis: enabled: false volumeSize: 2Gi openldap: enabled: false volumeSize: 2Gi minio: volumeSize: 20Gi monitoring: # type: external endpoint: http://prometheus-operated.kubesphere-monitoring-system.svc:9090 GPUMonitoring: enabled: false gpu: kinds: - resourceName: "nvidia.com/gpu" resourceType: "GPU" default: true es: # master: # volumeSize: 4Gi # replicas: 1 # resources: {} # data: # volumeSize: 20Gi # replicas: 1 # resources: {} logMaxAge: 7 elkPrefix: logstash basicAuth: enabled: false username: "" password: "" externalElasticsearchHost: "" externalElasticsearchPort: "" alerting: enabled: false # thanosruler: # replicas: 1 # resources: {} auditing: enabled: false # operator: # resources: {} # webhook: # resources: {} devops: enabled: false # resources: {} jenkinsMemoryLim: 8Gi jenkinsMemoryReq: 4Gi jenkinsVolumeSize: 8Gi events: enabled: false # operator: # resources: {} # exporter: # resources: {} # ruler: # enabled: true # replicas: 2 # resources: {} logging: enabled: false logsidecar: enabled: true replicas: 2 # resources: {} metrics_server: enabled: false monitoring: storageClass: "" node_exporter: port: 9100 # resources: {} # kube_rbac_proxy: # resources: {} # kube_state_metrics: # resources: {} # prometheus: # replicas: 1 # volumeSize: 20Gi # resources: {} # operator: # resources: {} # alertmanager: # replicas: 1 # resources: {} # notification_manager: # resources: {} # operator: # resources: {} # proxy: # resources: {} gpu: nvidia_dcgm_exporter: enabled: false # resources: {} multicluster: clusterRole: none network: networkpolicy: enabled: false ippool: type: none topology: type: none openpitrix: store: enabled: false servicemesh: enabled: false istio: components: ingressGateways: - name: istio-ingressgateway enabled: false cni: enabled: false edgeruntime: enabled: false kubeedge: enabled: false cloudCore: cloudHub: advertiseAddress: - "" service: cloudhubNodePort: "30000" cloudhubQuicNodePort: "30001" cloudhubHttpsNodePort: "30002" cloudstreamNodePort: "30003" tunnelNodePort: "30004" # resources: {} # hostNetWork: false iptables-manager: enabled: true mode: "external" # resources: {} # edgeService: # resources: {} terminal: timeout: 600
-
-
安装部署
./kk create cluster -f config-sample.yaml- 安装日志
kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l 'app in (ks-install, ks-installer)' -o jsonpath='{.items[0].metadata.name}') -f##################################################### ### Welcome to KubeSphere! ### ##################################################### Console: http://10.211.1.5:30880 Account: admin Password: P@88w0rd NOTES： 1. After you log into the console, please check the monitoring status of service components in "Cluster Management". If any service is not ready, please wait patiently until all components are up and running. 2. Please change the default password after login. ##################################################### https://kubesphere.io 2023-08-11 03:31:56 ##################################################### -
在 LBer 上为 30880端口 配置规则并在 ASG 上 permit
-
http://hostip：30880
-
-
测试
apiVersion: v1 kind: Service metadata: name: hello-kubernetes spec: type: NodePort ports: - port: 80 targetPort: 8080 selector: app: hello-kubernetes --- apiVersion: apps/v1 kind: Deployment metadata: name: hello-kubernetes spec: replicas: 3 selector: matchLabels: app: hello-kubernetes template: metadata: labels: app: hello-kubernetes spec: containers: - name: hello-kubernetes image: paulbouwer/hello-kubernetes:1.5 ports: - containerPort: 8080 env: - name: MESSAGE value: I just deployed a PodVM on the Azure VM Cluster!!-
配置外部访问30596 → 80
kubectl get svc NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE hello-kubernetes NodePort 10.233.31.15880:30596/TCP 167m -
负载均衡上配置
-
访问前端 公网 IP 或 DNS 名称
-