gitlab服务器502恢复过程

突然发现gitlab服务器无法进行访问

查看日志

# gitlab-ctl restart sidekiq 
ok: run: sidekiq: (pid 6278) 0s
# gitlab-ctl hup unicorn 
# gitlab-ctl status
run: gitlab-workhorse: (pid 3747) 168s; run: log: (pid 3342) 193s
run: logrotate: (pid 3762) 167s; run: log: (pid 3339) 193s
run: nginx: (pid 3768) 167s; run: log: (pid 3343) 193s
run: postgresql: (pid 3821) 166s; run: log: (pid 3336) 193s
run: redis: (pid 3829) 166s; run: log: (pid 3332) 193s
run: sidekiq: (pid 6278) 14s; run: log: (pid 3335) 193s
run: unicorn: (pid 6454) 1s; run: log: (pid 3333) 193s
# gitlab-ctl status
run: gitlab-workhorse: (pid 3747) 170s; run: log: (pid 3342) 195s
run: logrotate: (pid 3762) 169s; run: log: (pid 3339) 195s
run: nginx: (pid 3768) 169s; run: log: (pid 3343) 195s
run: postgresql: (pid 3821) 168s; run: log: (pid 3336) 195s
run: redis: (pid 3829) 168s; run: log: (pid 3332) 195s
run: sidekiq: (pid 6278) 16s; run: log: (pid 3335) 195s
run: unicorn: (pid 6486) 1s; run: log: (pid 3333) 195s
# gitlab-ctl status
run: gitlab-workhorse: (pid 3747) 177s; run: log: (pid 3342) 202s
run: logrotate: (pid 3762) 176s; run: log: (pid 3339) 202s
run: nginx: (pid 3768) 176s; run: log: (pid 3343) 202s
run: postgresql: (pid 3821) 175s; run: log: (pid 3336) 202s
run: redis: (pid 3829) 175s; run: log: (pid 3332) 202s
run: sidekiq: (pid 6278) 23s; run: log: (pid 3335) 202s
run: unicorn: (pid 6570) 2s; run: log: (pid 3333) 202s

发现unicorn进程不断变化。
查看日志

[root@cluster4 ~]# gitlab-ctl tail unicorn
==> /var/log/gitlab/unicorn/unicorn_stderr.log <==

==> /var/log/gitlab/unicorn/unicorn_stdout.log <==

==> /var/log/gitlab/unicorn/state <==

==> /var/log/gitlab/unicorn/current <==
2018-03-22_03:03:31.35843   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/configurator.rb:108:in `each'
2018-03-22_03:03:31.35843   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/configurator.rb:108:in `commit!'
2018-03-22_03:03:31.35844   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/http_server.rb:113:in `initialize'
2018-03-22_03:03:31.35844   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/bin/unicorn:126:in `new'
2018-03-22_03:03:31.35844   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/bin/unicorn:126:in `'
2018-03-22_03:03:31.35845   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/bin/unicorn:23:in `load'
2018-03-22_03:03:31.35845   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/bin/unicorn:23:in `
'
2018-03-22_03:03:31.37024 master failed to start, check stderr log for details
2018-03-22_03:03:32.37830 failed to start a new unicorn master
2018-03-22_03:03:32.39817 starting new unicorn master
2018-03-22_03:03:33.45858 /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/http_server.rb:778:in `initialize': Permission denied @ rb_sysopen - /var/log/gitlab/unicorn/unicorn_stderr.log (Errno::EACCES)
2018-03-22_03:03:33.45862   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/http_server.rb:778:in `open'
2018-03-22_03:03:33.45862   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/http_server.rb:778:in `redirect_io'
2018-03-22_03:03:33.45862   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/http_server.rb:175:in `stderr_path='
2018-03-22_03:03:33.45863   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/configurator.rb:111:in `block in commit!'
2018-03-22_03:03:33.45863   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/configurator.rb:108:in `each'
2018-03-22_03:03:33.45863   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/configurator.rb:108:in `commit!'
2018-03-22_03:03:33.45863   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/http_server.rb:113:in `initialize'
2018-03-22_03:03:33.45864   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/bin/unicorn:126:in `new'
2018-03-22_03:03:33.45865   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/bin/unicorn:126:in `'
2018-03-22_03:03:33.45865   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/bin/unicorn:23:in `load'
2018-03-22_03:03:33.45865   from /opt/gitlab/embedded/service/gem/ruby/2.1.0/bin/unicorn:23:in `
'
2018-03-22_03:03:33.46977 master failed to start, check stderr log for details
2018-03-22_03:03:34.47776 failed to start a new unicorn master
2018-03-22_03:03:34.49732 starting new unicorn master
2018-03-22_03:03:35.37668 /opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/http_server.rb:778:in `initialize': Permission denied @ rb_sysopen - /var/log/gitlab/unicorn/unicorn_stderr.log (Errno::EACCES)

发现一条
/opt/gitlab/embedded/service/gem/ruby/2.1.0/gems/unicorn-4.9.0/lib/unicorn/http_server.rb:778:in `initialize': Permission denied @ rb_sysopen - /var/log/gitlab/unicorn/unicorn_stderr.log (Errno::EACCES)
权限被拒绝

添加

# chmod -R 777 /var/log/gitlab
# gitlab-ctl tail unicorn
==> /var/log/gitlab/unicorn/unicorn_stderr.log <==
I, [2018-03-22T11:06:30.975318 #9107]  INFO -- : worker=28 ready
I, [2018-03-22T11:06:30.978437 #9110]  INFO -- : worker=29 spawned pid=9110
I, [2018-03-22T11:06:30.978861 #9110]  INFO -- : worker=29 ready
I, [2018-03-22T11:06:30.981850 #9113]  INFO -- : worker=30 spawned pid=9113
I, [2018-03-22T11:06:30.982292 #9113]  INFO -- : worker=30 ready
I, [2018-03-22T11:06:30.985443 #9116]  INFO -- : worker=31 spawned pid=9116
I, [2018-03-22T11:06:30.985850 #9116]  INFO -- : worker=31 ready
I, [2018-03-22T11:06:30.987826 #8946]  INFO -- : master process ready
I, [2018-03-22T11:06:30.988579 #9119]  INFO -- : worker=32 spawned pid=9119
I, [2018-03-22T11:06:30.988992 #9119]  INFO -- : worker=32 ready

==> /var/log/gitlab/unicorn/unicorn_stdout.log <==

发现进程正常，可以正常访问gitlab