[openshift 4.3] 裸金属离线部署

一. 部署架构说明


openshift 4.x 开始,部署方式不再依赖于3.x的ansible 部署方式,4.x开始部署方式有较大的改变,本次部署使用的是裸金属的部署方式,使用dhcp无人值守安装部署.



[openshift 4.3] 裸金属离线部署_第1张图片



# 部署流程说明

# 1.) 部署机,部署dns,dhcp,tftp,haproxy,

# 2.) 部署机,通过pxe的方式,部署出引导机,

# 3.) 引导机器部署完成之后,通过引导机,部署出openshift4.3集群


二. 部署环境规划

[openshift 4.3] 裸金属离线部署_第2张图片

三. 部署离线环境.(安装部署机)

a. ) 配置内部镜像仓库

#安装podman或docker,http-tools,http,

#yum -y install podman httpd httpd-tools

# 创建内部镜像仓库证书目录,数据目录,认证目录

# mkdir -p /opt/registry/{auth,certs,data}

# 生成tls 证书文件

# cd /opt/registry/certs

# 请注意证书的域名.

# openssl req -subj '/CN=registry.example.com/O=My Company Name LTD./C=US' -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout domain.key -out domain.crt

# 使用htpasswd 生成用户名密码文件

# htpasswd -bBc /opt/registry/auth/htpasswd test test

# 确保你的5000端口已在防火墙中放行

# 运行内部仓库镜像

# podman run --name poc-registry -p 5000:5000 \

-v /opt/registry/data:/var/lib/registry:z \

-v /opt/registry/auth:/auth:z \

-e "REGISTRY_AUTH=htpasswd" \

-e "REGISTRY_AUTH_HTPASSWD_REALM=Registry" \

-e "REGISTRY_HTTP_SECRET=ALongRandomSecretForRegistry" \

-e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \

-v /opt/registry/certs:/certs:z \

-e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \

-e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \

docker.io/library/registry:2

# 使用下面的命令验证内部仓库是否可以正常访问,这里应该返回一个空值,因为仓库中并没有镜像

# curl -u test:test -k https://registry.example.com:5000/v2/_catalog

# 如仓库无法访问,您应该检查

# 1.) registry 容器是否正常启动

# 2.) registry.example.com 域名是否可以正常解析

# 3.) htpasswd 文件中的用户名密码是否正确

# 4.) 防火墙5000端口是否开放

# 5.) 证书域名是否正确

b .) 获取离线镜像

# 下载oc客户端,您应该下载latest版本来部署你的openshift集群

# 请从当前页面下载 https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.3/

#https://cloud.redhat.com/openshift/install/pre-release获取pull-secret.json 秘钥

# 将本地镜像仓库中的验证信息使用base64 进行加密

# echo -n 'test:test' | base64 -w0

# 修改pull-secret.json 文件,将本地仓库验证信息加入

[openshift 4.3] 裸金属离线部署_第3张图片

# 设置环境变量

export OCP_RELEASE="4.3.0-x86_64"

# 本地镜像仓库域名端口

export LOCAL_REGISTRY='registry.example.com:5000'

# 本地镜像仓库存储库名称

export LOCAL_REPOSITORY='ocp4/openshift4'

# ocp发布的版本,一般不需要更改,也可以从https://mirror.openshift.com/pub/openshift-v4/clients/ocp-dev-preview/latest/release.txt获得

export PRODUCT_REPO='openshift-release-dev'

# 拉取ocp镜像的秘钥

export LOCAL_SECRET_JSON="./pull-secret-2.json"

# release版本名称,

export RELEASE_NAME="ocp-release"

# 将自签名的证书复制到默认信任证书路径

# cp /opt/registry/certs/domain.crt /etc/pki/ca-trust/source/anchors/

#

update-ca-trust extract

#使用 oc 命令拉取离线镜像,并推送到自己的内部仓库

oc adm -a ${LOCAL_SECRET_JSON} releasemirror \

    --from=quay.io/${PRODUCT_REPO}/${RELEASE_NAME}:${OCP_RELEASE} \

    --to=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY} \

    --to-release-image=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}

# 1.) pull镜像失败,大部分是超时引起的,需要从新执行脚本拉取镜像

# 2.) 每次拉取都会从头开始

# 离线部署的openshift-install 安装文件需要通过离线镜像创建,创建时需要连网# 载入前面创建镜像仓库的环境变量 

# oc adm -a ${LOCAL_SECRET_JSON} release extract --command=openshift-install "${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}"# cp openshift-install /usr/local/bin/


c.)  部署dnsmasq ,用于提供dns,dhcp ,tftp,服务

# 安装dnsmasq

# yum installdnsmasq tftp-server ipxe-bootimgs  -y

# systemctlenable dnsmasq && systemctl start dnsmasq

# 配置dnsmasq ,可以参考我的配置文件配置

# ############################

dhcp-range=192.168.1.173,192.168.1.180,255.255.248.0

enable-tftp

tftp-root=/var/lib/tftpboot

dhcp-match=set:bios,option:client-arch,0

dhcp-boot=tag:bios,undionly.kpxe

dhcp-match=set:efi32,option:client-arch,6

dhcp-boot=tag:efi32,ipxe.efi

dhcp-match=set:efibc,option:client-arch,7

dhcp-boot=tag:efibc,ipxe.efi

dhcp-match=set:efi64,option:client-arch,9

dhcp-boot=tag:efi64,ipxe.efi

dhcp-userclass=set:ipxe,iPXE

dhcp-option=option:router,192.168.1.1

dhcp-option=option:dns-server,192.168.1.171

dhcp-boot=tag:ipxe,http://bastion.ocp67.example.com:8080/boot.ipxe

address=/bastion.ocp67.example.com/192.168.1.171

address=/api.ocp67.example.com/192.168.1.171

address=/apps.ocp67.example.com/192.168.1.176

address=/api-int.ocp67.example.com/192.168.1.171

address=/master-0.ocp67.example.com/192.168.1.173

address=/etcd-0.ocp67.example.com/192.168.1.173

address=/master-1.ocp67.example.com/192.168.1.174

address=/etcd-1.ocp67.example.com/192.168.1.174

address=/master-2.ocp67.example.com/192.168.1.175

address=/etcd-2.ocp67.example.com/192.168.1.175

address=/node-0.ocp67.example.com/192.168.1.176

address=/node-1.ocp67.example.com/192.168.1.177

address=/node-2.ocp67.example.com/192.168.1.179

address=/bootstrap-0.ocp67.example.com/192.168.1.178

address=/registry.example.com/192.168.1.172

srv-host=_etcd-server-ssl._tcp.ocp67.example.com,etcd-0.ocp67.example.com,2380,10

srv-host=_etcd-server-ssl._tcp.ocp67.example.com,etcd-1.ocp67.example.com,2380,10

srv-host=_etcd-server-ssl._tcp.ocp67.example.com,etcd-2.ocp67.example.com,2380,10

dhcp-host=00:50:56:a2:39:71,master-0.ocp67.example.com,192.168.1.173

dhcp-host=00:50:56:a2:1b:bc,master-1.ocp67.example.com,192.168.1.174

dhcp-host=00:50:56:a2:1d:26,master-2.ocp67.example.com,192.168.1.175

dhcp-host=00:50:56:a2:25:f2,192.168.1.176

dhcp-host=00:50:56:a2:3f:4b,192.168.1.177

dhcp-host=00:50:56:a2:63:85,192.168.1.179

dhcp-host=00:50:56:a2:77:f0,192.168.1.178

log-queries

log-dhcp

####################################.

# 配置tftp server

# mkdir -p/var/lib/tftpboot

# ln -s /usr/share/ipxe/undionly.kpxe /var/lib/tftpboot

#开启防火墙策略,确保你的dns,dhcp,可以被正确访问到

# 启动dnsmasq

# systemctl enable dnsmasq && systemctl restart

dnsmasq


d.) 配置haproxy ,为api server,bootstarp ,提供负载均衡,

# 部署haproxy 负载均衡

# 该haproxy 主要是为master ,infra 节点提供负载均衡,可以使用其他负载均衡替代.

# 配置yum源,安装haproxy .

# yum install haproxy -y

# 配置haproxy 配置文件,增加master负载,infra负载

# ##############################

frontendk8s-int

        bind *:22623

        mode tcp

        default_backend app1

backend app1

        balance source

        mode tcp

        server bootstarp 192.168.1.178:22623 check

        server master-0 192.168.1.173:22623 check

        server master-1 192.168.1.174:22623 check

        server master-2 192.168.1.175:22623 check

frontend k8s

        bind *:6443

        mode tcp

        default_backend app2

backend app2

        balance source

        mode tcp

        server bootstarp 192.168.1.178:6443 check

        server master-0 192.168.1.173:6443 check

        server master-1 192.168.1.174:6443 check

        server master-2 192.168.1.175:6443 check


frontendinfra-0

        bind *:443

        mode tcp

        default_backend app3

backend app3

        balance source

        mode tcp

        server infra-0 192.168.1.177:443 check

frontendinfra-http

        bind *:80

        mode tcp

        default_backend app4

backend app4

        balance source

        mode tcp

        server infra-0 192.168.1.177:80 check


####################################

# 启动haproxy ,确保6443,22623,80,443 端口处于haproxy监听状态

# systemctlenable haproxy && systemctl start haproxy

e.) 配置matchbox,该服务主要是用作pxe安装时,分配ign部署配置文件

# 下载matchbox,可以到github的release页面下载, https://github.com/poseidon/matchbox/releases

# 解压 matchbox-v0.8.3-linux-amd64.tar.gz 安装包

# tar xf matchbox-v0.8.3-linux-amd64.tar.gz

# cdmatchbox-v0.8.3-linux-amd64/

# cp matchbox/usr/local/bin

# cp contrib/systemd/matchbox-local.service/etc/systemd/system/matchbox.service

# mkdir -p/var/lib/matchbox/{assets,groups,ignition,profiles}

# cd /var/lib/matchbox/assets

# 下载rhcos 安装文件,你可以从这里下载: https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/pre-release/latest/

# 需要下载:

# 1.) rhcos-43.81.201912030353.0-installer-initramfs.x86_64.img

# 2.) rhcos-43.81.201912030353.0-installer-kernel-x86_64

# 3.) rhcos-43.81.201912030353.0-metal.x86_64.raw.gz

# 配置pxe自动部署配置文件

# /var/lib/matchbox/profiles/bootstrap.json

##################################

{

"id": "bootstrap",

"name": "OCP 4 – Bootstrap",

"ignition_id": "bootstrap.ign",

"boot": {

"kernel": "/assets/rhcos-4.1.0-x86_64-installer-kernel",

"initrd": [

"/assets/rhcos-4.1.0-x86_64-installer-initramfs.img"

],

"args": [

"ip=dhcp",

"rd.neednet=1",

"console=tty0",

"console=ttyS0",

"coreos.inst=yes",

"coreos.inst.install_dev=sda",

"coreos.inst.image_url=http://bastion.ocp67.example.com:8080/assets/rhcos-4.1.0-x86_64-metal-bios.raw.gz",

"coreos.inst.ignition_url=http://bastion.ocp67.example.com:8080/ignition?mac=${mac:hexhyp}"

]

}

}

# /var/lib/matchbox/profiles/master.json

{

"id": "master",

"name": "OCP 4 – Master",

"ignition_id": "master.ign",

"boot": {

"kernel": "/assets/rhcos-4.1.0-x86_64-installer-kernel",

"initrd": [

"/assets/rhcos-4.1.0-x86_64-installer-initramfs.img"

],

"args": [

"ip=dhcp",

"rd.neednet=1",

"console=tty0",

"console=ttyS0",

"coreos.inst=yes",

"coreos.inst.install_dev=sda",

"coreos.inst.image_url=http://bastion.ocp67.example.com:8080/assets/rhcos-4.1.0-x86_64-metal-bios.raw.gz",

"coreos.inst.ignition_url=http://bastion.ocp67.example.com:8080/ignition?mac=${mac:hexhyp}"

]

}

}

##############

# /var/lib/matchbox/profiles/infnod.json

{

"id": "infnod",

"name": "OCP 4 – Infrastructure Node",

"ignition_id": "worker.ign",

"boot": {

"kernel": "/assets/rhcos-4.1.0-x86_64-installer-kernel",

"initrd": [

"/assets/rhcos-4.1.0-x86_64-installer-initramfs.img"

],

"args": [

"ip=dhcp",

"rd.neednet=1",

"console=tty0",

"console=ttyS0",

"coreos.inst=yes",

"coreos.inst.install_dev=sda",

"coreos.inst.image_url=http://bastion.ocp67.example.com:8080/assets/rhcos-4.1.0-x86_64-metal-bios.raw.gz",

"coreos.inst.ignition_url=http://bastion.ocp67.example.com:8080/ignition?mac=${mac:hexhyp}"

]

}

}

# /var/lib/matchbox/profiles/cptnod.json

{

"id": "cptnod",

"name": "OCP 4 – Compute Node",

"ignition_id": "worker.ign",

"boot": {

"kernel": "/assets/rhcos-4.1.0-x86_64-installer-kernel",

"initrd": [

"/assets/rhcos-4.1.0-x86_64-installer-initramfs.img"

],

"args": [

"ip=dhcp",

"rd.neednet=1",

"console=tty0",

"console=ttyS0",

"coreos.inst=yes",

"coreos.inst.install_dev=sda",

"coreos.inst.image_url=http://bastion.ocp67.example.com:8080/assets/rhcos-4.1.0-x86_64-metal-bios.raw.gz",

"coreos.inst.ignition_url=http://bastion.ocp67.example.com:8080/ignition?mac=${mac:hexhyp}"

]

}

}

# /var/lib/matchbox/groups/bootstrap-0.json

{

"id": "bootstrap-0",

"name": "OCP 4 – Bootstrap server",

"profile": "bootstrap",

"selector": {

"mac": "00:50:56:a2:77:f0"

}

}

# /var/lib/matchbox/groups/master-{0-2}.json

{

"id": "master-1",

"name": "OCP 4 – Master 2",

"profile": "master",

"selector": {

"mac": "00:50:56:a2:1b:bc"

}

}



{

"id": "master-2",

"name": "OCP 4 – Master 3",

"profile": "master",

"selector": {

"mac": "00:50:56:a2:1d:26"

}

}



{

"id": "master-0",

"name": "OCP 4 – Master 1",

"profile": "master",

"selector": {

"mac": "00:50:56:a2:39:71"

}

}

#/var/lib/matchbox/groups/infnod-0.json

{

"id": "infnod-0",

"name": "OCP 4 – Infrastructure Node #1",

"profile": "infnod",

"selector": {

"mac": "00:50:56:a2:25:f2"

}

}



# /var/lib/matchbox/groups/cptnod-{0-1}.json

{

"id": "cptnod-0",

"name": "OCP 4 – Compute node #1",

"profile": "cptnod",

"selector": {

"mac": "00:50:56:a2:3f:4b"

}

}



{

"id": "cptnod-1",

"name": "OCP 4 – Compute node #2",

"profile": "cptnod",

"selector": {

"mac": "00:50:56:a2:63:85"

}

}


[openshift 4.3] 裸金属离线部署_第4张图片

#########################

# 1.) 以上是配置matchbox 自动化部署配置的配置文件,主要需要修改mac地址,需要先确认相应的mac地址的角色,

# 2.) groups中的配置文件“profile”:

“cptnod”,需要与profile中定义的名称一致.

# 启动matchbox

# systemctlenable matchbox && systemctl start matchbox

# matchbox监听8080端口,请确保在防火墙中打开此端口

四. 安装openshift集群

# 创建一对公钥与私钥,用作登录rhcos ,

# ssh-keygen -trsa -b 2048 -N "" -f /root/.ssh/id_ras

# mkdir/root/ocp4

# 创建安装集群配置文件 install-config.yaml

# 请注意修改红色部分

####################

apiVersion: v1

baseDomain:example.com

compute:

-hyperthreading: Enabled

  name: worker

  replicas: 0

controlPlane:

  hyperthreading: Enabled

  name: master

  replicas: 3

metadata:

  name: ocp67

networking:

  clusterNetworks:

  - cidr: 10.254.0.0/16

    hostPrefix: 24

  networkType: OpenShiftSDN

  serviceNetwork:

  - 172.30.0.0/16

platform:

  none: {}

pullSecret: '{"auths":{"registry.example.com:5000":

{"auth": "ZHVtbXk6ZHVtbXk=","email":

"noemail@localhost"}}}'

sshKey: '...cat /root/.ssh/id_rsa.pub ... ssh 公钥'

additionalTrustBundle:|

  -----BEGIN CERTIFICATE-----

  ... cat /etc/pki/ca-trust/source/anchors/domain.crt

  ...私有镜像仓库证书的server.crt

  -----END CERTIFICATE-----

imageContentSources:

- mirrors:

  -registry.example.com:5000/ocp4/openshift4

  source:quay.io/openshift-release-dev/ocp-release

- mirrors:

  -registry.example.com:5000/ocp4/openshift4

  source:quay.io/openshift-release-dev/ocp-v4.0-art-dev

# 创建集群安装部署配置文件,文件创建之后install-config.yml 文件会被删除,所以建议备份一下.

# openshift-install create ignition-configs


[openshift 4.3] 裸金属离线部署_第5张图片


####################

# 复制 *.ign文件到matchbox 的部署目录中

# cp *.ign/var/lib/matchbox/ignition

# 从网卡启动服务器安装openshift 4.2

# 1.) 启动bootstarp-0 机器,安装bootstarp

# openshift-install --dir=/root/ocp4 wait-forbootstrap-complete –log-level debug

# 2.) 等待bootstarp 安装完毕,检查服务是否正常.登录bootstarp,检查服务端口6443,22623.

# 3.) 启动 master-0-3 安装,master,etcd 集群.

# 4.) 将haproxy 中的bootstarp 负载移除,并重启haproxy,

# oc patch configs.imageregistry.operator.openshift.iocluster \

–type merge \

–patch '{"spec":{"storage":{"emptyDir":{}}}}'


# openshift-install --dir=/root/ocp4 wait-forinstall-complete –log-level debug


# 5.) 启动node节点,来部署node~

#6.) 等待node节点部署完毕,需要在部署机,批准node节点申请集群证书的请求


# oc get csr  

# 没有批准的证书申请请求会处于pending状态,请确认。

# oc adm certificate approve

注意

由于 CSR 会自动轮转,因此请在将机器添加到集群后一小时内批准您的 CSR。如

果没有在一小时内批准,证书将会轮转,每个节点将会存在多个证书。您必须批准

所有这些证书。批准初始 CSR 后,集群的 kube-controller-manager 会自动批准

后续的节点客户端 CSR。您必须实施一个方法来自动批准 kubelet 提供的证书请

# 等待所有的operator 上线~

# oc get no 


[openshift 4.3] 裸金属离线部署_第6张图片

你可能感兴趣的:([openshift 4.3] 裸金属离线部署)