golang平滑重启库overseer实现原理
overseer主要完成了三部分功能：
1、连接的无损关闭，2、连接的平滑重启，3、文件变更的自动重启。
下面依次讲一下：
一、连接的无损关闭
golang官方的net包是不支持连接的无损关闭的，当主监听协程退出时，并不会等待各个实际work协程的处理完成。
以下是golang官方代码：
Go/src/net/http/server.go
func (srv *Server) Serve(l net.Listener) error {
if fn := testHookServerServe; fn != nil {
fn(srv, l) // call hook with unwrapped listener
}
origListener := l
l = &onceCloseListener{Listener: l}
defer l.Close()
if err := srv.setupHTTP2_Serve(); err != nil {
return err
}
if !srv.trackListener(&l, true) {
return ErrServerClosed
}
defer srv.trackListener(&l, false)
baseCtx := context.Background()
if srv.BaseContext != nil {
baseCtx = srv.BaseContext(origListener)
if baseCtx == nil {
panic("BaseContext returned a nil context")
}
}
var tempDelay time.Duration // how long to sleep on accept failure
ctx := context.WithValue(baseCtx, ServerContextKey, srv)
for {
rw, err := l.Accept()
if err != nil {
if srv.shuttingDown() {
return ErrServerClosed
}
if ne, ok := err.(net.Error); ok && ne.Temporary() {
if tempDelay == 0 {
tempDelay = 5 * time.Millisecond
} else {
tempDelay *= 2
}
if max := 1 * time.Second; tempDelay > max {
tempDelay = max
}
srv.logf("http: Accept error: %v; retrying in %v", err, tempDelay)
time.Sleep(tempDelay)
continue
}
return err
}
connCtx := ctx
if cc := srv.ConnContext; cc != nil {
connCtx = cc(connCtx, rw)
if connCtx == nil {
panic("ConnContext returned nil")
}
}
tempDelay = 0
c := srv.newConn(rw)
c.setState(c.rwc, StateNew, runHooks) // before Serve can return
go c.serve(connCtx)
}
}
当监听套接字关闭，l.Accept()退出循环时，并不会等待go c.serve(connCtx)协程的处理完成。
overseer的处理方式是，包装了golang的监听套接字和连接套接字，通过sync.WaitGroup提供了对主协程异步等待work协程处理完成的支持。
overseer代码如下：
overseer-v1.1.6\graceful.go
func (l *overseerListener) Accept() (net.Conn, error) {
conn, err := l.Listener.(*net.TCPListener).AcceptTCP()
if err != nil {
return nil, err
}
conn.SetKeepAlive(true) // see http.tcpKeepAliveListener
conn.SetKeepAlivePeriod(3 * time.Minute) // see http.tcpKeepAliveListener
uconn := overseerConn{
Conn: conn,
wg: &l.wg,
closed: make(chan bool),
}
go func() {
//connection watcher
select {
case <-l.closeByForce:
uconn.Close()
case <-uconn.closed:
//closed manually
}
}()
l.wg.Add(1)
return uconn, nil
}
//non-blocking trigger close
func (l *overseerListener) release(timeout time.Duration) {
//stop accepting connections - release fd
l.closeError = l.Listener.Close()
//start timer, close by force if deadline not met
waited := make(chan bool)
go func() {
l.wg.Wait()
waited <- true
}()
go func() {
select {
case <-time.After(timeout):
close(l.closeByForce)
case <-waited:
//no need to force close
}
}()
}
//blocking wait for close
func (l *overseerListener) Close() error {
l.wg.Wait()
return l.closeError
}
func (o overseerConn) Close() error {
err := o.Conn.Close()
if err == nil {
o.wg.Done()
o.closed <- true
}
return err
}
在(l *overseerListener) Accept函数中，每生成一个work连接，执行l.wg.Add(1)，在(o overseerConn) Close函数中，每关闭一个work连接，执行o.wg.Done()。
在异步关闭模式(l *overseerListener) release函数中和在同步关闭模式(l *overseerListener) Close函数中都会调用l.wg.Wait()以等待work协程的处理完成。
监听套接字关闭流程：
1、work进程收到重启信号，或者master进程收到重启信号然后转发到work进程。
2、work进程的信号处理里包含对(l *overseerListener) release的调用。
3、在(l *overseerListener) release里关闭监听套接字，并异步l.wg.Wait()。
4、在官方包net/http/server.go的 (srv *Server) Serve里l.Accept()出错返回，退出监听循环，然后执行defer l.Close()，即(l *overseerListener) Close。
5、在(l *overseerListener) Close里同步执行l.wg.Wait()，等待work连接处理完成。
6、work连接处理完成时，会调用(o overseerConn) Close()，进而调用o.wg.Done()。
7、所有work连接处理完成后，向master进程发送SIGUSR1信号。
8、master进程收到SIGUSR1信号后，将true写入mp.descriptorsReleased管道。
9、master进程的(mp *master) fork里，收到mp.descriptorsReleased后，结束本次fork，进入下一次fork。
二、连接的平滑重启
所谓平滑重启，就是重启不会造成客户端的断连，对客户端无感知，比如原有的排队连接不会被丢弃，所以监听套接字通过master进程在新旧work进程间传递，而不是新启的work进程重新创建监听连接。
监听套接字由master进程创建：
overseer-v1.1.6/proc_master.go
func (mp *master) retreiveFileDescriptors() error {
mp.slaveExtraFiles = make([]*os.File, len(mp.Config.Addresses))
for i, addr := range mp.Config.Addresses {
a, err := net.ResolveTCPAddr("tcp", addr)
if err != nil {
return fmt.Errorf("Invalid address %s (%s)", addr, err)
}
l, err := net.ListenTCP("tcp", a)
if err != nil {
return err
}
f, err := l.File()
if err != nil {
return fmt.Errorf("Failed to retreive fd for: %s (%s)", addr, err)
}
if err := l.Close(); err != nil {
return fmt.Errorf("Failed to close listener for: %s (%s)", addr, err)
}
mp.slaveExtraFiles[i] = f
}
return nil
}从mp.Config.Addresses中拿到地址，建立监听连接，最后把文件句柄存入mp.slaveExtraFiles。
在这个过程中调用了(l *TCPListener) Close，但其实对work进程无影响，影响的只是master进程自己不能读写监听套接字。
这里引用下对网络套接字close和shutdown的区别：
close ---- 关闭本进程的socket id，但连接还是开着的，用这个socket id的其它进程还能用这个连接，能读或写这个socket id。
shutdown ---- 则破坏了socket 连接，读的时候可能侦探到EOF结束符，写的时候可能会收到一个SIGPIPE信号，这个信号可能直到socket buffer被填充了才收到，shutdown还有一个关闭方式的参数，0 不能再读，1不能再写，2 读写都不能。
将mp.slaveExtraFiles传递给子进程即work进程：
overseer-v1.1.6/proc_master.go
func (mp *master) fork() error {
mp.debugf("starting %s", mp.binPath)
cmd := exec.Command(mp.binPath)
//mark this new process as the "active" slave process.
//this process is assumed to be holding the socket files.
mp.slaveCmd = cmd
mp.slaveID++
//provide the slave process with some state
e := os.Environ()
e = append(e, envBinID+"="+hex.EncodeToString(mp.binHash))
e = append(e, envBinPath+"="+mp.binPath)
e = append(e, envSlaveID+"="+strconv.Itoa(mp.slaveID))
e = append(e, envIsSlave+"=1")
e = append(e, envNumFDs+"="+strconv.Itoa(len(mp.slaveExtraFiles)))
cmd.Env = e
//inherit master args/stdfiles
cmd.Args = os.Args
cmd.Stdin = os.Stdin
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
//include socket files
cmd.ExtraFiles = mp.slaveExtraFiles
if err := cmd.Start(); err != nil {
return fmt.Errorf("Failed to start slave process: %s", err)
}
//was scheduled to restart, notify success
if mp.restarting {
mp.restartedAt = time.Now()
mp.restarting = false
mp.restarted <- true
}
//convert wait into channel
cmdwait := make(chan error)
go func() {
cmdwait <- cmd.Wait()
}()
//wait....
select {
case err := <-cmdwait:
//program exited before releasing descriptors
//proxy exit code out to master
code := 0
if err != nil {
code = 1
if exiterr, ok := err.(*exec.ExitError); ok {
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
code = status.ExitStatus()
}
}
}
mp.debugf("prog exited with %d", code)
//if a restarts are disabled or if it was an
//unexpected crash, proxy this exit straight
//through to the main process
if mp.NoRestart || !mp.restarting {
os.Exit(code)
}
case <-mp.descriptorsReleased:
//if descriptors are released, the program
//has yielded control of its sockets and
//a parallel instance of the program can be
//started safely. it should serve state.Listeners
//to ensure downtime is kept at <1sec. The previous
//cmd.Wait() will still be consumed though the
//result will be discarded.
}
return nil
}
通过cmd.ExtraFiles = mp.slaveExtraFiles语句向子进程传递套接字，这个参数最终传递给fork系统调用，传递的fd会被子进程继承。
子进程即work进程处理继承的套接字：
overseer-v1.1.6/proc_slave.go
func (sp *slave) run() error {
sp.id = os.Getenv(envSlaveID)
sp.debugf("run")
sp.state.Enabled = true
sp.state.ID = os.Getenv(envBinID)
sp.state.StartedAt = time.Now()
sp.state.Address = sp.Config.Address
sp.state.Addresses = sp.Config.Addresses
sp.state.GracefulShutdown = make(chan bool, 1)
sp.state.BinPath = os.Getenv(envBinPath)
if err := sp.watchParent(); err != nil {
return err
}
if err := sp.initFileDescriptors(); err != nil {
return err
}
sp.watchSignal()
//run program with state
sp.debugf("start program")
sp.Config.Program(sp.state)
return nil
}
func (sp *slave) initFileDescriptors() error {
//inspect file descriptors
numFDs, err := strconv.Atoi(os.Getenv(envNumFDs))
if err != nil {
return fmt.Errorf("invalid %s integer", envNumFDs)
}
sp.listeners = make([]*overseerListener, numFDs)
sp.state.Listeners = make([]net.Listener, numFDs)
for i := 0; i < numFDs; i++ {
f := os.NewFile(uintptr(3+i), "")
l, err := net.FileListener(f)
if err != nil {
return fmt.Errorf("failed to inherit file descriptor: %d", i)
}
u := newOverseerListener(l)
sp.listeners[i] = u
sp.state.Listeners[i] = u
}
if len(sp.state.Listeners) > 0 {
sp.state.Listener = sp.state.Listeners[0]
}
return nil
}子进程只是重新包装套接字，并没有新建监听连接，包装成u := newOverseerListener(l)类型，这些监听套接字最后传递给sp.Config.Program(sp.state)，即用户的启动程序：
overseer-v1.1.6/example/main.go
// convert your 'main()' into a 'prog(state)'
// 'prog()' is run in a child process
func prog(state overseer.State) {
fmt.Printf("app#%s (%s) listening...\n", BuildID, state.ID)
http.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
d, _ := time.ParseDuration(r.URL.Query().Get("d"))
time.Sleep(d)
fmt.Fprintf(w, "app#%s (%s) %v says hello\n", BuildID, state.ID, state.StartedAt)
}))
http.Serve(state.Listener, nil)
fmt.Printf("app#%s (%s) exiting...\n", BuildID, state.ID)
}
// then create another 'main' which runs the upgrades
// 'main()' is run in the initial process
func main() {
overseer.Run(overseer.Config{
Program: prog,
Address: ":5001",
Fetcher: &fetcher.File{Path: "my_app_next"},
Debug: true, //display log of overseer actions
TerminateTimeout: 10 * time.Minute,
})
}在用户程序中http.Serve(state.Listener, nil)调用：
1、使用的accept方式是包装后的(l *overseerListener) Accept()。
2、defer l.Close()使用也是包装后的(l *overseerListener) Close()。
3、由(l *overseerListener) Accept()创建的work连接也都包装成了overseerConn连接，在关闭时会调用(o overseerConn) Close()
三、文件变更的自动重启
能够自动监视文件变化，有变更时自动触发重启流程。
在master进程启动时检查配置，如果设置了mp.Config.Fetcher则进入fetchLoop：
overseer-v1.1.6/proc_master.go
// fetchLoop is run in a goroutine
func (mp *master) fetchLoop() {
min := mp.Config.MinFetchInterval
time.Sleep(min)
for {
t0 := time.Now()
mp.fetch()
//duration fetch of fetch
diff := time.Now().Sub(t0)
if diff < min {
delay := min - diff
//ensures at least MinFetchInterval delay.
//should be throttled by the fetcher!
time.Sleep(delay)
}
}
}mp.Config.MinFetchInterval默认是1秒，也就是每秒检查一次变更。time.Duration类型，可以设置更小的粒度。
已经支持的fetcher包括：fetcher_file.go、fetcher_github.go、fetcher_http.go、fetcher_s3.go。
以fetcher_file.go为例说明。
1、文件变更的判断：
overseer-v1.1.6/proc_master.go
//tee off to sha1
hash := sha1.New()
reader = io.TeeReader(reader, hash)
//write to a temp file
_, err = io.Copy(tmpBin, reader)
if err != nil {
mp.warnf("failed to write temp binary: %s", err)
return
}
//compare hash
newHash := hash.Sum(nil)
if bytes.Equal(mp.binHash, newHash) {
mp.debugf("hash match - skip")
return
}通过sha1算法实现，比较新旧hash值，并没有关注文件时间戳。
2、验证是可执行文件，且是支持overseer的：
overseer-v1.1.6/proc_master.go
tokenIn := token()
cmd := exec.Command(tmpBinPath)
cmd.Env = append(os.Environ(), []string{envBinCheck + "=" + tokenIn}...)
cmd.Args = os.Args
returned := false
go func() {
time.Sleep(5 * time.Second)
if !returned {
mp.warnf("sanity check against fetched executable timed-out, check overseer is running")
if cmd.Process != nil {
cmd.Process.Kill()
}
}
}()
tokenOut, err := cmd.CombinedOutput()
returned = true
if err != nil {
mp.warnf("failed to run temp binary: %s (%s) output \"%s\"", err, tmpBinPath, tokenOut)
return
}
if tokenIn != string(tokenOut) {
mp.warnf("sanity check failed")
return
}这是通过overseer预埋的代码实现的：
overseer-v1.1.6/overseer.go
//sanityCheck returns true if a check was performed
func sanityCheck() bool {
//sanity check
if token := os.Getenv(envBinCheck); token != "" {
fmt.Fprint(os.Stdout, token)
return true
}
//legacy sanity check using old env var
if token := os.Getenv(envBinCheckLegacy); token != "" {
fmt.Fprint(os.Stdout, token)
return true
}
return false
}
这段代码在main启动时在overseer.Run里会调用到，传递固定的环境变量，然后命令行输出会原样显示出来即为成功。
3、覆盖旧文件，并触发重启。
overseer-v1.1.6/proc_master.go
//overwrite!
if err := overwrite(mp.binPath, tmpBinPath); err != nil {
mp.warnf("failed to overwrite binary: %s", err)
return
}
mp.debugf("upgraded binary (%x -> %x)", mp.binHash[:12], newHash[:12])
mp.binHash = newHash
//binary successfully replaced
if !mp.Config.NoRestartAfterFetch {
mp.triggerRestart()
}由(mp *master) triggerRestart进入重启流程：
overseer-v1.1.6/proc_master.go
func (mp *master) triggerRestart() {
if mp.restarting {
mp.debugf("already graceful restarting")
return //skip
} else if mp.slaveCmd == nil || mp.restarting {
mp.debugf("no slave process")
return //skip
}
mp.debugf("graceful restart triggered")
mp.restarting = true
mp.awaitingUSR1 = true
mp.signalledAt = time.Now()
mp.sendSignal(mp.Config.RestartSignal) //ask nicely to terminate
select {
case <-mp.restarted:
//success
mp.debugf("restart success")
case <-time.After(mp.TerminateTimeout):
//times up mr. process, we did ask nicely!
mp.debugf("graceful timeout, forcing exit")
mp.sendSignal(os.Kill)
}
}向子进程发送mp.Config.RestartSignal信号，子进程收到信号后，关闭监听套接字然后向父进程发送SIGUSR1信号：
overseer-v1.1.6/proc_slave.go
if len(sp.listeners) > 0 {
//perform graceful shutdown
for _, l := range sp.listeners {
l.release(sp.Config.TerminateTimeout)
}
//signal release of held sockets, allows master to start
//a new process before this child has actually exited.
//early restarts not supported with restarts disabled.
if !sp.NoRestart {
sp.masterProc.Signal(SIGUSR1)
}
//listeners should be waiting on connections to close...
}父进程收到SIGUSR1信号后，通知mp.descriptorsReleased管道监听套接字已经关闭：
overseer-v1.1.6/proc_master.go
//**during a restart** a SIGUSR1 signals
//to the master process that, the file
//descriptors have been released
if mp.awaitingUSR1 && s == SIGUSR1 {
mp.debugf("signaled, sockets ready")
mp.awaitingUSR1 = false
mp.descriptorsReleased <- true
} else最终回到(mp *master) fork函数，fork函数一直在等待mp.descriptorsReleased通知或者cmd.Wait子进程退出，收到管道通知后fork退出，进入下一轮fork循环。
overseer-v1.1.6/proc_master.go
func (mp *master) fork() error {
//... ...
//... ...
//... ...
//convert wait into channel
cmdwait := make(chan error)
go func() {
cmdwait <- cmd.Wait()
}()
//wait....
select {
case err := <-cmdwait:
//program exited before releasing descriptors
//proxy exit code out to master
code := 0
if err != nil {
code = 1
if exiterr, ok := err.(*exec.ExitError); ok {
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
code = status.ExitStatus()
}
}
}
mp.debugf("prog exited with %d", code)
//if a restarts are disabled or if it was an
//unexpected crash, proxy this exit straight
//through to the main process
if mp.NoRestart || !mp.restarting {
os.Exit(code)
}
case <-mp.descriptorsReleased:
//if descriptors are released, the program
//has yielded control of its sockets and
//a parallel instance of the program can be
//started safely. it should serve state.Listeners
//to ensure downtime is kept at <1sec. The previous
//cmd.Wait() will still be consumed though the
//result will be discarded.
}
return nil
}
--end--