Gitlab 部署在Mesos 容器云平台 存储采用ceph 变成无状态容器实践

从单台虚拟机启动一个gitlab 服务器是很容易的事情,
GITLAB_HOME=pwd/data/gitlab
docker run -d
–hostname gitlab
–publish 8443:443 --publish 80:80 --publish 2222:22
–name gitlab
–restart always
–volume $GITLAB_HOME/config:/etc/gitlab
–volume $GITLAB_HOME/logs:/var/log/gitlab
–volume $GITLAB_HOME/data:/var/opt/gitlab
gitlab/gitlab-ce

上面的做法虽然简单，但是存在几个问题:
1.必须要依赖虚拟机，虚拟机挂了，这个服务器就挂了.
2.数据不安全 存在虚拟机本地，虚拟机坏了，数据就没了.
3.缺乏位置透明性 要记住是部署在哪台虚拟机之上。

能不能部署在容器云平台之上，这样，随时随地，想要就有，可以直接通过域名访问，不关心gitlab 在哪里。

通过一番实验，终于成功的，把gitlab 实例部署在mesos+ceph 平台之上。

前提: Mesos+Marathon 平台正常
安装有mesos-dns bind-dns
Marathon 上的json文件如下:

     {

“id”: “/devops/gitlab”,
“cmd”: null,
“cpus”: 2,
“mem”: 8192,
“disk”: 0,
“instances”: 1,
“acceptedResourceRoles”: [
“*”
],
“container”: {
“type”: “DOCKER”,
“docker”: {
“forcePullImage”: false,
“image”: “gitlab/gitlab-ce:11.7.4-ce.0”,
“parameters”: [
{
“key”: “dns”,
“value”: “10.20.5.31”
},
{
“key”: “network”,
“value”: “frontend”
},
{
“key”: “volume-driver”,
“value”: “rexray”
},
{
“key”: “volume”,
“value”: “gitlab-config:/etc/gitlab”
},
{
“key”: “volume”,
“value”: “gitlab-data:/var/opt/gitlab”
},
{
“key”: “volume”,
“value”: “gitlab-log:/var/log/gitlab”
}
],
“privileged”: true
},
“volumes”: []
},
“env”: {
“TZ”: “Asia/Shanghai”
},
“healthChecks”: [
{
“gracePeriodSeconds”: 600,
“intervalSeconds”: 60,
“maxConsecutiveFailures”: 10,
“path”: “/”,
“port”: 80,
“protocol”: “MESOS_HTTP”,
“ipProtocol”: “IPv4”,
“timeoutSeconds”: 60,
“delaySeconds”: 15
}
],
“portDefinitions”: []
}

启动成功后，在mesos-agent3上，可以看到gitlab 自动创建了3个rbd 镜像

这里要特别强调的是: 1.内存和CPU 不能设置太小,gitlab容量里好多服务，很费资源

2. 必须设置 “privileged”: true 这样才能保证权限足够。这个点和单机启动确实不一样。
3. Marathon健康检查尽量设置长一点, 因为gitlab 第一次启动需要耗费很长时间初始化，拷贝文件，太短了，marathon会把正在初始化的gitlab docker 实例执行超时关闭。
   

[root@mesos-agent3 ~]# docker inspect 098a3922f3a0
[
{
“Id”: “098a3922f3a08c06030d7a67e31906e60273b816afe8ae0502c450667e6a513d”,
“Created”: “2019-02-20T02:16:19.674800159Z”,
“Path”: “/assets/wrapper”,
“Args”: [],
“State”: {
“Status”: “running”,
“Running”: true,
“Paused”: false,
“Restarting”: false,
“OOMKilled”: false,
“Dead”: false,
“Pid”: 12020,
“ExitCode”: 0,
“Error”: “”,
“StartedAt”: “2019-02-20T02:16:35.598206224Z”,
“FinishedAt”: “0001-01-01T00:00:00Z”,
“Health”: {
“Status”: “healthy”,
“FailingStreak”: 0,
“Log”: [
{
“Start”: “2019-02-20T10:35:11.563481232+08:00”,
“End”: “2019-02-20T10:35:13.06968516+08:00”,
“ExitCode”: 0,
“Output”: " % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:–:-- --:–:-- --:–:-- 0\r 0 0 0 0 0 0 0 0 --:–:-- --:–:-- --:–:-- 0\r 0 0 0 0 0 0 0 0 --:–:-- 0:00:01 --:–:-- 0\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\nHelp · GitLab\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n }
]
}
},
“Image”: “sha256:a4ef8b071476eb0237ea5ddd5a695cd80d9e00b7a5ff80e0ec15ba56512021f3”,
“ResolvConfPath”: “/var/lib/docker/containers/098a3922f3a08c06030d7a67e31906e60273b816afe8ae0502c450667e6a513d/resolv.conf”,
“HostnamePath”: “/var/lib/docker/containers/098a3922f3a08c06030d7a67e31906e60273b816afe8ae0502c450667e6a513d/hostname”,
“HostsPath”: “/var/lib/docker/containers/098a3922f3a08c06030d7a67e31906e60273b816afe8ae0502c450667e6a513d/hosts”,
“LogPath”: “/var/lib/docker/containers/098a3922f3a08c06030d7a67e31906e60273b816afe8ae0502c450667e6a513d/098a3922f3a08c06030d7a67e31906e60273b816afe8ae0502c450667e6a513d-json.log”,
“Name”: “/mesos-97962239-b2ca-4d56-9c29-3015226967d2”,
“RestartCount”: 0,
“Driver”: “overlay2”,
“Platform”: “linux”,
“MountLabel”: “”,
“ProcessLabel”: “”,
“AppArmorProfile”: “”,
“ExecIDs”: null,
“HostConfig”: {
“Binds”: [
“gitlab-data:/var/opt/gitlab”,
“gitlab-log:/var/log/gitlab”,
“/var/lib/mesos/slaves/adec85b1-ea50-4c7c-a9a0-4645dcdecc4b-S1/frameworks/f0c0c535-6e7c-48d8-996f-918e2a1c4757-0000/executors/devops_gitlab.691a0aba-34b5-11e9-86cb-86fa233b14af/runs/97962239-b2ca-4d56-9c29-3015226967d2:/mnt/mesos/sandbox”,
“gitlab-config:/etc/gitlab”
],
“ContainerIDFile”: “”,
“LogConfig”: {
“Type”: “json-file”,
“Config”: {}
},
“NetworkMode”: “frontend”,
“PortBindings”: {},
“RestartPolicy”: {
“Name”: “no”,
“MaximumRetryCount”: 0
},
“AutoRemove”: false,
“VolumeDriver”: “rexray”,
“VolumesFrom”: null,
“CapAdd”: null,
“CapDrop”: null,
“Dns”: [
“10.20.5.31”
],
“DnsOptions”: [],
“DnsSearch”: [],
“ExtraHosts”: null,
“GroupAdd”: null,
“IpcMode”: “shareable”,
“Cgroup”: “”,
“Links”: null,
“OomScoreAdj”: 0,
“PidMode”: “”,
“Privileged”: true,
“PublishAllPorts”: false,
“ReadonlyRootfs”: false,
“SecurityOpt”: [
“label=disable”
],
“UTSMode”: “”,
“UsernsMode”: “”,
“ShmSize”: 67108864,
“Runtime”: “runc”,
“ConsoleSize”: [
0,
0
],
“Isolation”: “”,
“CpuShares”: 2048,
“Memory”: 8589934592,
“NanoCpus”: 0,
“CgroupParent”: “”,
“BlkioWeight”: 0,
“BlkioWeightDevice”: [],
“BlkioDeviceReadBps”: null,
“BlkioDeviceWriteBps”: null,
“BlkioDeviceReadIOps”: null,
“BlkioDeviceWriteIOps”: null,
“CpuPeriod”: 0,
“CpuQuota”: 0,
“CpuRealtimePeriod”: 0,
“CpuRealtimeRuntime”: 0,
“CpusetCpus”: “”,
“CpusetMems”: “”,
“Devices”: [],
“DeviceCgroupRules”: null,
“DiskQuota”: 0,
“KernelMemory”: 0,
“MemoryReservation”: 0,
“MemorySwap”: 17179869184,
“MemorySwappiness”: null,
“OomKillDisable”: false,
“PidsLimit”: 0,
“Ulimits”: null,
“CpuCount”: 0,
“CpuPercent”: 0,
“IOMaximumIOps”: 0,
“IOMaximumBandwidth”: 0,
“MaskedPaths”: null,
“ReadonlyPaths”: null
},
“GraphDriver”: {
“Data”: {
“LowerDir”: “/var/lib/docker/overlay2/7691c2c03e8159e39fe379e98307253b076314c1b29c097334fd5be11bcbfc79-init/diff:/var/lib/docker/overlay2/a9c63ce8a2939340be9f596fe189fc1f9340b3ce321ded6300301cff993eed95/diff:/var/lib/docker/overlay2/5b776ae7bbf286f220cc78df4f95df90d3b267469c9e75fd6b4121851e09641d/diff:/var/lib/docker/overlay2/a11ffe5ff9561185772bd2f537c3b07c8536a747643145bea4f0f15800ee149e/diff:/var/lib/docker/overlay2/f326f7471aa5b21dbf860d968347cf7759e43fae197c5af32a2ea1d06c1d7825/diff:/var/lib/docker/overlay2/aada5eb3912bb79068d3d172db98d4e0acb19bd977e64fe25f5b8e51ec9c660f/diff:/var/lib/docker/overlay2/7adde31af706db66aaf60f5cc7950dc6e7f58e09c79ac9aba4d21caf766db07b/diff:/var/lib/docker/overlay2/ed99a1e6f9515b1ad80c45ea1353380de94de0f8dd1cee9963439efbc3cae830/diff:/var/lib/docker/overlay2/91a3bb9dfbd3231c16b51abd9642221daa50fbcb1a82a98fa3d5b42c650c3939/diff:/var/lib/docker/overlay2/5672697fc48d6e786f91b5502c3889662c550a46a40f068d154ceb02b023f4ab/diff:/var/lib/docker/overlay2/fa573861521122946de83923df5c2075dc6e602809609b3c3e969a83fad21dd4/diff”,
“MergedDir”: “/var/lib/docker/overlay2/7691c2c03e8159e39fe379e98307253b076314c1b29c097334fd5be11bcbfc79/merged”,
“UpperDir”: “/var/lib/docker/overlay2/7691c2c03e8159e39fe379e98307253b076314c1b29c097334fd5be11bcbfc79/diff”,
“WorkDir”: “/var/lib/docker/overlay2/7691c2c03e8159e39fe379e98307253b076314c1b29c097334fd5be11bcbfc79/work”
},
“Name”: “overlay2”
},
“Mounts”: [
{
“Type”: “volume”,
“Name”: “gitlab-data”,
“Source”: “”,
“Destination”: “/var/opt/gitlab”,
“Driver”: “rexray”,
“Mode”: “”,
“RW”: true,
“Propagation”: “”
},
{
“Type”: “volume”,
“Name”: “gitlab-log”,
“Source”: “”,
“Destination”: “/var/log/gitlab”,
“Driver”: “rexray”,
“Mode”: “”,
“RW”: true,
“Propagation”: “”
},
{
“Type”: “bind”,
“Source”: “/var/lib/mesos/slaves/adec85b1-ea50-4c7c-a9a0-4645dcdecc4b-S1/frameworks/f0c0c535-6e7c-48d8-996f-918e2a1c4757-0000/executors/devops_gitlab.691a0aba-34b5-11e9-86cb-86fa233b14af/runs/97962239-b2ca-4d56-9c29-3015226967d2”,
“Destination”: “/mnt/mesos/sandbox”,
“Mode”: “”,
“RW”: true,
“Propagation”: “rprivate”
},
{
“Type”: “volume”,
“Name”: “gitlab-config”,
“Source”: “”,
“Destination”: “/etc/gitlab”,
“Driver”: “rexray”,
“Mode”: “”,
“RW”: true,
“Propagation”: “”
}
],
“Config”: {
“Hostname”: “098a3922f3a0”,
“Domainname”: “”,
“User”: “”,
“AttachStdin”: false,
“AttachStdout”: true,
“AttachStderr”: true,
“ExposedPorts”: {
“22/tcp”: {},
“443/tcp”: {},
“80/tcp”: {}
},
“Tty”: false,
“OpenStdin”: false,
“StdinOnce”: false,
“Env”: [
“MARATHON_APP_DOCKER_IMAGE=gitlab/gitlab-ce:11.7.4-ce.0”,
“MARATHON_APP_ID=/devops/gitlab”,
“MARATHON_APP_LABELS=”,
“MARATHON_APP_RESOURCE_CPUS=2.0”,
“MARATHON_APP_RESOURCE_DISK=0.0”,
“MARATHON_APP_RESOURCE_MEM=8192.0”,
“TZ=Asia/Shanghai”,
“HOST=mesos-agent3.cityworks.cn”,
“MARATHON_APP_RESOURCE_GPUS=0”,
“MARATHON_APP_VERSION=2019-02-20T02:15:36.798Z”,
“MESOS_CONTAINER_NAME=mesos-97962239-b2ca-4d56-9c29-3015226967d2”,
“MESOS_SANDBOX=/mnt/mesos/sandbox”,
“MESOS_TASK_ID=devops_gitlab.691a0aba-34b5-11e9-86cb-86fa233b14af”,
“PATH=/opt/gitlab/embedded/bin:/opt/gitlab/bin:/assets:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin”,
“TERM=xterm”
],
“Cmd”: [
“/assets/wrapper”
],
“Healthcheck”: {
“Test”: [
“CMD-SHELL”,
“/opt/gitlab/bin/gitlab-healthcheck --fail --max-time 10”
],
“Interval”: 60000000000,
“Timeout”: 30000000000,
“Retries”: 5
},
“ArgsEscaped”: true,
“Image”: “gitlab/gitlab-ce:11.7.4-ce.0”,
“Volumes”: {
“/etc/gitlab”: {},
“/var/log/gitlab”: {},
“/var/opt/gitlab”: {}
},
“WorkingDir”: “”,
“Entrypoint”: null,
“OnBuild”: null,
“Labels”: {
“MESOS_TASK_ID”: “devops_gitlab.691a0aba-34b5-11e9-86cb-86fa233b14af”
}
},
“NetworkSettings”: {
“Bridge”: “”,
“SandboxID”: “4036b8ecab877727881bba9cdf24cabcb7edbd1822760738f2a32ebbe36b83fd”,
“HairpinMode”: false,
“LinkLocalIPv6Address”: “”,
“LinkLocalIPv6PrefixLen”: 0,
“Ports”: {},
“SandboxKey”: “/var/run/docker/netns/4036b8ecab87”,
“SecondaryIPAddresses”: null,
“SecondaryIPv6Addresses”: null,
“EndpointID”: “”,
“Gateway”: “”,
“GlobalIPv6Address”: “”,
“GlobalIPv6PrefixLen”: 0,
“IPAddress”: “”,
“IPPrefixLen”: 0,
“IPv6Gateway”: “”,
“MacAddress”: “”,
“Networks”: {
“frontend”: {
“IPAMConfig”: null,
“Links”: null,
“Aliases”: [
“098a3922f3a0”
],
“NetworkID”: “2dcc846aecff8b1d782d545c3791359a1a1b7359b89b34a611faffca2010a315”,
“EndpointID”: “e9e2d3696bdc31496f154f4e95fb8537e5b35de810c05842ecd90f83cc8f2adb”,
“Gateway”: “169.254.1.1”,
“IPAddress”: “172.30.102.84”,
“IPPrefixLen”: 32,
“IPv6Gateway”: “”,
“GlobalIPv6Address”: “”,
“GlobalIPv6PrefixLen”: 0,
“MacAddress”: “ee:ee:ee:ee:ee:ee”,
“DriverOpts”: null
}
}
}
}
]
[root@mesos-agent3 ~]#