k8s集群部署harbor镜像仓库
目录
一 安装部署
部署环境
二安装docker， docker-compose
三解压Harbor文件
四配置HTTPS证书
五配置Harbor YML文件
六运行安装程序脚本
七测试命令登陆仓库 https://192.168.100.13
1，登录harbor
2，然后重启docker.service 重启Harbor
3，登录私有仓库
4，创建k8s项目
5，推送命令参考页面
八Harbor仓库里开始推送镜像给k8s
1，镜像表示已经标记在项目
2， 推送镜像到项目中查看结果
3， 将镜像推送至k8s集群里 完成
一 安装部署
部署环境
- 安装Docker
- 安装 docker-compose
- 下载harbor上传主机 （harbor-offline-installer-v2.1.1.tgz）
| 官网 | 地址 |
|---|---|
| harbor | Releases · goharbor/harbor · GitHub |
二安装docker， docker-compose
#安装docker-ce
yum install docker-ce docker-compose
#启动docker服务
systemctl start docker
#查看docker版本
docker -v
三解压Harbor文件
#创建解压目录harbor
mkdir harbor
#进入解压目录harbor
cd harbor
#解压压缩包到当前目录下的harbor目录中
tar -zxvf harbor-offline-installer-v2.1.1.tgz
[root@node1 ~]# mkdir harbor
[root@node1 ~]# cd harbor/
[root@node1 harbor]#
[root@node1 harbor]# rz -E
rz waiting to receive.
[root@node1 harbor]# ls
harbor-offline-installer-v2.1.1.tgz
[root@node1 harbor]# tar -zxvf harbor-offline-installer-v2.1.1.tgz
harbor/harbor.v2.1.1.tar.gz
harbor/prepare
harbor/LICENSE
harbor/install.sh
harbor/common.sh
harbor/harbor.yml.tmpl
[root@node1 harbor]# ls
harbor harbor-offline-installer-v2.1.1.tgz
四配置HTTPS证书
[root@node1 harbor]# openssl genrsa -out ca.key 4096
Generating RSA private key, 4096 bit long modulus
...................................................................................................++
.........................................................................................................................++
e is 65537 (0x10001)
[root@node1 harbor]# openssl req -x509 -new -nodes -sha512 -days 3650 -subj "/C=CN/ST=shanxi/L=shanxi/O=example/OU=Personal/CN=192.168.100.13" -key ca.key -out ca.crt
[root@node1 harbor]#
[root@node1 harbor]# mkdir -p /data/cert 创建data/cert 路径
[root@node1 harbor]# cp ca.crt /data/cert
[root@node1 harbor]# cp ca.key /data/cert
[root@node1 harbor]#
[root@node1 harbor]# systemctl restart docker五配置Harbor YML文件
[root@node1 harbor]# cd harbor/
[root@node1 harbor]# ls
common.sh harbor.v2.1.1.tar.gz harbor.yml.tmpl install.sh LICENSE prepare
[root@node1 harbor]# mv harbor.yml.tmpl harbor.yml
[root@node1 harbor]# ls
common.sh harbor.v2.1.1.tar.gz harbor.yml input install.sh LICENSE prepare
[root@node1 harbor]# vim harbor.yml
# Configuration file of Harbor
# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: 192.168.100.13 ##更改域名
# http related config
#http: ##注销http
# port for http, default is 80. If https enabled, this port will redirect to https port
# port: 80 ##注销端口
# https related config
https:
# https port for harbor, default is 443
port: 443
# The path of cert and key files for nginx
certificate: /data/cert/ca.crt ##更改路径证书私钥
private_key: /data/cert/ca.key ##更改路径证书私钥
六运行安装程序脚本
./prepare
./install.sh
结果
[root@node1 harbor]# ./prepare
prepare base dir is set to /root/harbor/harbor
Clearing the configuration file: /config/registry/passwd
Clearing the configuration file: /config/registry/root.crt
Clearing the configuration file: /config/registry/config.yml
Clearing the configuration file: /config/nginx/nginx.conf
Clearing the configuration file: /config/portal/nginx.conf
Clearing the configuration file: /config/db/env
Clearing the configuration file: /config/core/app.conf
Clearing the configuration file: /config/core/env
Clearing the configuration file: /config/log/rsyslog_docker.conf
Clearing the configuration file: /config/log/logrotate.conf
Clearing the configuration file: /config/registryctl/env
Clearing the configuration file: /config/registryctl/config.yml
Clearing the configuration file: /config/jobservice/env
Clearing the configuration file: /config/jobservice/config.yml
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
loaded secret from file: /data/secret/keys/secretkey
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
[root@node1 harbor]# ./install.sh
[Step 0]: checking if docker is installed ...
Note: docker version: 23.0.1
[Step 1]: checking docker-compose is installed ...
Note: docker-compose version: 1.18.0
[Step 2]: loading Harbor images ...
Loaded image: goharbor/harbor-db:v2.1.1
Loaded image: goharbor/redis-photon:v2.1.1
Loaded image: goharbor/notary-signer-photon:v2.1.1
Loaded image: goharbor/clair-photon:v2.1.1
Loaded image: goharbor/clair-adapter-photon:v2.1.1
Loaded image: goharbor/harbor-core:v2.1.1
Loaded image: goharbor/harbor-portal:v2.1.1
Loaded image: goharbor/harbor-log:v2.1.1
Loaded image: goharbor/nginx-photon:v2.1.1
Loaded image: goharbor/prepare:v2.1.1
Loaded image: goharbor/harbor-registryctl:v2.1.1
Loaded image: goharbor/notary-server-photon:v2.1.1
Loaded image: goharbor/trivy-adapter-photon:v2.1.1
Loaded image: goharbor/harbor-jobservice:v2.1.1
Loaded image: goharbor/chartmuseum-photon:v2.1.1
Loaded image: goharbor/registry-photon:v2.1.1
[Step 3]: preparing environment ...
[Step 4]: preparing harbor configs ...
prepare base dir is set to /root/harbor/harbor
Clearing the configuration file: /config/registry/passwd
Clearing the configuration file: /config/registry/config.yml
Clearing the configuration file: /config/nginx/nginx.conf
Clearing the configuration file: /config/portal/nginx.conf
Clearing the configuration file: /config/db/env
Clearing the configuration file: /config/core/app.conf
Clearing the configuration file: /config/core/env
Clearing the configuration file: /config/log/rsyslog_docker.conf
Clearing the configuration file: /config/log/logrotate.conf
Clearing the configuration file: /config/registryctl/env
Clearing the configuration file: /config/registryctl/config.yml
Clearing the configuration file: /config/jobservice/env
Clearing the configuration file: /config/jobservice/config.yml
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
loaded secret from file: /data/secret/keys/secretkey
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
Note: stopping existing Harbor instance ...
Stopping harbor-jobservice ... done
Stopping nginx ... done
Stopping harbor-core ... done
Stopping harbor-portal ... done
Stopping registryctl ... done
Stopping redis ... done
Stopping harbor-db ... done
Stopping registry ... done
Stopping harbor-log ... done
Removing harbor-jobservice ... done
Removing nginx ... done
Removing harbor-core ... done
Removing harbor-portal ... done
Removing registryctl ... done
Removing redis ... done
Creating harbor-log ... done
Removing registry ... done
Removing harbor-log ... done
Removing network harbor_harbor
Creating redis ... done
Creating harbor-core ... done
Creating network "harbor_harbor" with the default driver
Creating nginx ... done
Creating registry ...
Creating redis ...
Creating harbor-db ...
Creating registryctl ...
Creating harbor-portal ...
Creating harbor-core ...
Creating nginx ...
Creating harbor-jobservice ...
✔ ----Harbor has been installed and started successfully.----
七测试命令登陆仓库 https://192.168.100.13
登录账号密码在harbor.yml 文件里
admin
Harbor12345
harbor_admin_password: Harbor12345
1，登录harbor
[root@node1 harbor]# docker login 192.168.100.13
Username: admin
Password:
Error response from daemon: Get https://192.168.100.13/v2/: x509: cannot validate certificate for 192.168.100.13 because it doesn't contain any IP SANs
如果出现这种问题
解决办法：
在/usr/lib/systemd/system/docker.service中添加私有仓库地址
找到service，注释后的第一行ExecStart参数最后添加--insecure-registry 私有仓库地址
vim /usr/lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket containerd.service
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --insecure-registry https://192.168.100.13 ##添加地址
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
2，然后重启docker.service 重启Harbor
systemctl daemon-reload
systemctl restart docker.service
在harbor的路径下重启harbor服务
[root@node1 harbor]# ./install.sh
查看harbor服务是否起来
[root@node1 harbor]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS
c9d645513951 goharbor/harbor-jobservice:v2.1.1 "/harbor/entrypoint.…" 31 minutes ago Up 6 seconds (h
794442077e1a goharbor/nginx-photon:v2.1.1 "nginx -g 'daemon of…" 31 minutes ago Up 14 minutes (
194045c4afa9 goharbor/harbor-core:v2.1.1 "/harbor/entrypoint.…" 31 minutes ago Restarting (2)
58c345de0490 goharbor/redis-photon:v2.1.1 "redis-server /etc/r…" 31 minutes ago Exited (128) 17
067bca17a96a goharbor/registry-photon:v2.1.1 "/home/harbor/entryp…" 31 minutes ago Up 14 minutes (
9779f1e938fc goharbor/harbor-portal:v2.1.1 "nginx -g 'daemon of…" 31 minutes ago Up 14 minutes (
c21da2e479ac goharbor/harbor-registryctl:v2.1.1 "/home/harbor/start.…" 31 minutes ago Up 14 minutes (
3e1b1261f174 goharbor/harbor-db:v2.1.1 "/docker-entrypoint.…" 31 minutes ago Up 14 minutes (
6795fad40dab goharbor/harbor-log:v2.1.1 "/bin/sh -c /usr/loc…" 31 minutes ago Up 14 minutes (
看到有2个服务还没有up起来
重新启动下
[root@node1 harbor]# docker start 194045c4afa9
194045c4afa9
[root@node1 harbor]# docker start 58c345de0490
58c345de0490
[root@node1 harbor]#
再次查看服务已经全部起来
[root@node1 harbor]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
b7aad9d19036 goharbor/harbor-jobservice:v2.1.1 "/harbor/entrypoint.…" 24 minutes ago Up 24 minutes (healthy) harbor-jobservice
5901942fffd5 goharbor/nginx-photon:v2.1.1 "nginx -g 'daemon of…" 24 minutes ago Up 24 minutes (healthy) 0.0.0.0:80->8080/tcp, 0.0.0.0:443->8443/tcp nginx
a552eec595ba goharbor/harbor-core:v2.1.1 "/harbor/entrypoint.…" 24 minutes ago Up 24 minutes (healthy) harbor-core
8b8b0e786e2d goharbor/redis-photon:v2.1.1 "redis-server /etc/r…" 24 minutes ago Up 24 minutes (healthy) redis
b5ff29631d8d goharbor/registry-photon:v2.1.1 "/home/harbor/entryp…" 24 minutes ago Up 24 minutes (healthy) registry
663a0715b40f goharbor/harbor-registryctl:v2.1.1 "/home/harbor/start.…" 24 minutes ago Up 24 minutes (healthy) registryctl
8d801e424c50 goharbor/harbor-portal:v2.1.1 "nginx -g 'daemon of…" 24 minutes ago Up 24 minutes (healthy) harbor-portal
b2907672f756 goharbor/harbor-db:v2.1.1 "/docker-entrypoint.…" 24 minutes ago Up 24 minutes (healthy) harbor-db
3fc51d7606aa goharbor/harbor-log:v2.1.1 "/bin/sh -c /usr/loc…" 24 minutes ago Up 24 minutes (healthy) 127.0.0.1:1514->10514/tcp harbor-log
重新启动harbor
[root@node1 harbor]# ./install.sh
3，登录私有仓库
再次登录成功
[root@node1 harbor]# docker login 192.168.100.13
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@node1 harbor]# 4，创建k8s项目
5，推送命令参考页面
在项目中标记镜像：
docker tag SOURCE_IMAGE[:TAG] 192.168.100.13/k8s/IMAGE[:TAG]
推送镜像到当前项目：
docker push 192.168.100.13/k8s/IMAGE[:TAG]
八Harbor仓库里开始推送镜像给k8s
docker tag nginx:latest 192.168.100.13/k8s/nginx:latest
docker push 192.168.100.13/k8s/nginx:latest
1，镜像表示已经标记在项目
2， 推送镜像到项目中查看结果
3， 将镜像推送至k8s集群里 完成
先删除之前k8s推送的Nginx镜像
kubectl delete svc nginx
删除创建的deployment
kubectl delete deployment nginx
通过命令远程拉取harbor的镜像并创建pod
kubectl run my-nginx --image=192.168.100.13/k8s/nginx:latest --port=80
创建deployment my-nginx
kubectl create deployment my-nginx --image=my-nginx
为刚创建的服务nginx扩容
kubectl scale deployment/my-nginx --replicas=3
删除pods
kubectl delete pods my-nginx --grace-period=0 --force
删除deployment的pods
kubectl delete deployment my-nginx
[root@node1 harbor]# docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
flannel/flannel v0.21.3 0d004b381af6 8 days ago 64.1MB
nginx latest 904b8cb13b93 2 weeks ago 142MB
192.168.100.13/k8s/nginx latest 904b8cb13b93 2 weeks ago 142MB
redis latest f9c173b0f012 2 weeks ago
[root@node1 harbor]# kubectl delete svc nginx
service "nginx" deleted
[root@node1 harbor]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.10.0.1 443/TCP 3d1h
mysql NodePort 10.10.66.181 3064:32262/TCP 17h
redis NodePort 10.10.20.119 81:31803/TCP 17h
[root@master1 ~]# kubectl delete deployment nginx
deployment.apps "nginx" deleted
[root@master1 ~]# kubectl get pod
NAME READY STATUS RESTARTS AGE
redis-8464b6fbc9-sj6tm 1/1 Running 0 18h
[root@node1 harbor]# kubectl get pods
NAME READY STATUS RESTARTS AGE
redis-8464b6fbc9-sj6tm 1/1 Running 0 17h
[root@master1 ~]# kubectl delete deployment nginx
deployment.apps "nginx" deleted
[root@master1 ~]# kubectl get pod
NAME READY STATUS RESTARTS AGE
my-nginx 1/1 Running 0 33m
redis-8464b6fbc9-sj6tm 1/1 Running 0 18h
[root@node1 harbor]# kubectl run my-nginx --image=192.168.100.13/k8s/nginx:latest --port=80
pod/my-nginx created
[root@node1 harbor]# kubectl get pods
NAME READY STATUS RESTARTS AGE
my-nginx 1/1 Running 0 34m
redis-8464b6fbc9-sj6tm 1/1 Running 0 18h
[root@node1 harbor]# kubectl create deployment my-nginx --image=my-nginx
deployment.apps/my-nginx created
[root@node1 harbor]#
[root@node1 harbor]# kubectl get pods
NAME READY STATUS RESTARTS AGE
my-nginx 1/1 Running 0 43m
my-nginx-8796bf77c-h6cfn 0/1 ContainerCreating 0 6s
redis-8464b6fbc9-sj6tm 1/1 Running 0 18h
[root@node1 harbor]# kubectl scale deployment/my-nginx --replicas=3
deployment.apps/my-nginx scaled
[root@node1 harbor]#
[root@node1 harbor]# kubectl get pods
NAME READY STATUS RESTARTS AGE
my-nginx 1/1 Running 0 48m
my-nginx-8796bf77c-gp8kb 0/1 ContainerCreating 0 3s
my-nginx-8796bf77c-h6cfn 0/1 ContainerCreating 0 5m23s
my-nginx-8796bf77c-zg4bp 0/1 ContainerCreating 0 3s
redis-8464b6fbc9-sj6tm 1/1 Running 0 18h
[root@master1 ~]# kubectl get deployment
NAME READY UP-TO-DATE AVAILABLE AGE
my-nginx 0/3 3 0 10m
redis 1/1 1 1 18h
[root@master1 ~]# kubectl delete pods my-nginx --grace-period=0 --force
warning: Immediate deletion does not wait for confirmation that the running resource has been terminated. The resource may continue to run on the cluster indefinitely.
pod "my-nginx" force deleted
[root@master1 ~]#
[root@master1 ~]# kubectl delete deployment my-nginx
deployment.apps "my-nginx" deleted
[root@master1 ~]#
[root@master1 ~]# kubectl get pod
NAME READY STATUS RESTARTS AGE
redis-8464b6fbc9-sj6tm 1/1 Running 1 (21m ago) 21h