我们知道一般在调用http client后都会close Response.Body,如下:
client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
下面我们来看下为什么resp.Body需要Close,一定需要Close吗？
我们先通过"net/http/httptrace"来验证下:
1.不使用Close
代码:
package main
import (
"fmt"
"net/http"
"net/http/httptrace"
)
func main() {
req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
if err != nil {
panic(err)
}
trace := &httptrace.ClientTrace{
GetConn: func(hostPort string) {
fmt.Println("GetConn:", hostPort)
},
GotConn: func(connInfo httptrace.GotConnInfo) {
fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
},
PutIdleConn: func(err error) {
fmt.Printf("PutIdleConn err=%v\n", err)
},
}
req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
client := &http.Client{
Transport: &http.Transport{
MaxIdleConns: 1, //最大Idle
MaxConnsPerHost: 2, //每个host最大conns
}}
for i := 1; i <= 10; i++ {
fmt.Printf("==============第[%d]次请求================\n", i)
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(resp.Status)
}
}输出:
==============第[1]次请求================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55131
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============第[2]次请求================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:55132
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============第[3]次请求================
GetConn: www.baidu.com:443
结论:
我们设置了MaxConnsPerHost=2,由于没有close导致没有释放连接，执行两次请求后就卡住了，不能继续向下执行。并且第一次和第二次请求连接没有复用
2.只使用Close,不读取resp.Body
代码:
package main
import (
"fmt"
"net/http"
"net/http/httptrace"
)
func main() {
req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
if err != nil {
panic(err)
}
trace := &httptrace.ClientTrace{
GetConn: func(hostPort string) {
fmt.Println("GetConn:", hostPort)
},
GotConn: func(connInfo httptrace.GotConnInfo) {
fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
},
PutIdleConn: func(err error) {
fmt.Printf("PutIdleConn err=%v\n", err)
},
}
req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
client := &http.Client{
Transport: &http.Transport{
MaxIdleConns: 1, //最大Idle
MaxConnsPerHost: 2, //每个host最大conns
}}
for i := 1; i <= 10; i++ {
fmt.Printf("==============第[%d]次请求================\n", i)
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
resp.Body.Close() //注意这里close了
fmt.Println(resp.Status)
}
}输出:
==============1================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54948
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============2================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54949
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============3================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54950
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============4================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54954
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============5================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54955
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============6================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54956
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============7================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54957
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============8================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54958
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============9================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54959
GotConn remoteAddr: 183.232.231.172:443
200 OK
==============10================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:54960
GotConn remoteAddr: 183.232.231.172:443
200 OK结论：
虽然可以继续执行，说明连接释放了。但GotConn信息eused=false WasIdle=false并且每次localAddr都是不同的,我们发现每次请求获得的连接都是新申请的。
3.只读取resp.Body,不使用Close()
我们知道resp.Body实现了io.Reader接口方法,我们通常的做法就通过调用Read()方法来读取内容。
代码:
package main
import (
"fmt"
"io"
"net/http"
"net/http/httptrace"
)
func main() {
req, err := http.NewRequest("GET", "https://www.baidu.com/", nil)
if err != nil {
panic(err)
}
trace := &httptrace.ClientTrace{
GetConn: func(hostPort string) {
fmt.Println("GetConn:", hostPort)
},
GotConn: func(connInfo httptrace.GotConnInfo) {
fmt.Printf("GotConn Reused=%v WasIdle=%v IdleTime=%v\n", connInfo.Reused, connInfo.WasIdle, connInfo.IdleTime)
fmt.Printf("GotConn localAddr: %s\n", connInfo.Conn.LocalAddr())
fmt.Printf("GotConn remoteAddr: %s\n", connInfo.Conn.RemoteAddr())
},
PutIdleConn: func(err error) {
fmt.Printf("PutIdleConn err=%v\n", err)
},
}
req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace))
client := &http.Client{
Transport: &http.Transport{
MaxIdleConns: 1, //最大Idle
MaxConnsPerHost: 2, //每个host最大conns
}}
for i := 1; i <= 10; i++ {
fmt.Printf("==============第[%d]次请求================\n", i)
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
io.ReadAll(resp.Body) //读取body里面的内容
fmt.Println(resp.Status)
}
}输出:
==============第[1]次请求================
GetConn: www.baidu.com:443
GotConn Reused=false WasIdle=false IdleTime=0s
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK
==============第[2]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=116.037µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK
==============第[3]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=72.154µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK
==============第[4]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=60.102µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK
==============第[5]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=71.807µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK
==============第[6]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.616µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK
==============第[7]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=47.205µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK
==============第[8]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=74.207µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK
==============第[9]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=52.414µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK
==============第[10]次请求================
GetConn: www.baidu.com:443
GotConn Reused=true WasIdle=true IdleTime=81.137µs
GotConn localAddr: 192.168.10.104:56450
GotConn remoteAddr: 183.232.231.174:443
PutIdleConn err=
200 OK 结论:
我们可以看下每次输出的GotConn信息Reused=true WasIdle=true并且localAddr都是相同的。说明每次执行完后都把连接放回了idleConn pool里面，下去获取连接时可以从idleConn pool里面获取。
源码分析
(c Client) Do(req Request)-->(t Transport) roundTrip(req Request)-->t.getConn(treq, cm)-->t.queueForDial(w)->go t.dialConnFor(w)-->go readLoop()
我们readLoop()源码来分析，readLoop()是循环读取连接内容的方法:
go1.18 net/http/transport.go
(pc *persistConn) readLoop()是一个持久化连接在不停的读取conn里面的内容，下面我跟随源码看一下为什么要close/readAll
func (pc *persistConn) readLoop() {
closeErr := errReadLoopExiting // default value, if not changed below
defer func() {
pc.close(closeErr)
pc.t.removeIdleConn(pc)
}()
// tryPutIdleConn就是把连接放回IdleConn然后让连接可以复用
tryPutIdleConn := func(trace *httptrace.ClientTrace) bool {
if err := pc.t.tryPutIdleConn(pc); err != nil {
closeErr = err
if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled {
trace.PutIdleConn(err)
}
return false
}
if trace != nil && trace.PutIdleConn != nil {
trace.PutIdleConn(nil)
}
return true
}
// eofc is used to block caller goroutines reading from Response.Body
// at EOF until this goroutines has (potentially) added the connection
// back to the idle pool.
eofc := make(chan struct{})
defer close(eofc) // unblock reader on errors
// Read this once, before loop starts. (to avoid races in tests)
testHookMu.Lock()
testHookReadLoopBeforeNextRead := testHookReadLoopBeforeNextRead
testHookMu.Unlock()
//这里的alive用来判断是否继续读取连接内容
alive := true
for alive {
pc.readLimit = pc.maxHeaderResponseSize()
_, err := pc.br.Peek(1)
pc.mu.Lock()
if pc.numExpectedResponses == 0 {
pc.readLoopPeekFailLocked(err)
pc.mu.Unlock()
return
}
pc.mu.Unlock()
rc := <-pc.reqch
trace := httptrace.ContextClientTrace(rc.req.Context())
var resp *Response
if err == nil {
resp, err = pc.readResponse(rc, trace)
} else {
err = transportReadFromServerError{err}
closeErr = err
}
if err != nil {
if pc.readLimit <= 0 {
err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize())
}
select {
case rc.ch <- responseAndError{err: err}:
case <-rc.callerGone:
return
}
return
}
pc.readLimit = maxInt64 // effectively no limit for response bodies
pc.mu.Lock()
pc.numExpectedResponses--
pc.mu.Unlock()
bodyWritable := resp.bodyIsWritable()
hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0
if resp.Close || rc.req.Close || resp.StatusCode <= 199 || bodyWritable {
// Don't do keep-alive on error if either party requested a close
// or we get an unexpected informational (1xx) response.
// StatusCode 100 is already handled above.
alive = false
}
......
waitForBodyRead := make(chan bool, 2)
//最重要的就是这个把resp.body通过bodyEOFSignal封装生成新的resp.Body，
//下面会讲到为什么通过bodyEOFSignal封装
body := &bodyEOFSignal{
body: resp.Body,
//如果调用earlyCloseFn就执行waitForBodyRead <- false
earlyCloseFn: func() error {
waitForBodyRead <- false
<-eofc // will be closed by deferred call at the end of the function
return nil
},
//如果调用fn()并且 err == io.EOF 就执行waitForBodyRead <- true
fn: func(err error) error {
isEOF := err == io.EOF
waitForBodyRead <- isEOF
if isEOF {
<-eofc // see comment above eofc declaration
} else if err != nil {
if cerr := pc.canceled(); cerr != nil {
return cerr
}
}
return err
},
}
// 重新把封装成bodyEOFSignal的body赋值给resp.Body
resp.Body = body
if rc.addedGzip && ascii.EqualFold(resp.Header.Get("Content-Encoding"), "gzip") {
resp.Body = &gzipReader{body: body}
resp.Header.Del("Content-Encoding")
resp.Header.Del("Content-Length")
resp.ContentLength = -1
resp.Uncompressed = true
}
select {
// 把resp推送到rc.ch然后在roundTrip()返回给外部
case rc.ch <- responseAndError{res: resp}:
case <-rc.callerGone:
return
}
// Before looping back to the top of this function and peeking on
// the bufio.Reader, wait for the caller goroutine to finish
// reading the response body. (or for cancellation or death)
select {
// 这里是最重要的,从waitForBodyRead阻塞获取bodyEof
case bodyEOF := <-waitForBodyRead:
replaced := pc.t.replaceReqCanceler(rc.cancelKey, nil) // before pc might return to idle pool
//这里比较巧妙的是如果alive && bodyEOF && !pc.sawEOF && pc.wroteRequest()
//才会执行tryPutIdleConn(trace),就是把连接放回idleConn,这就是为什么使用close()是关闭了连接但是没有复用，而使用ReadAll()确复用了idle。
//因为使用使用ReadAll后返回的bodyEOF=true而直接使用Close()返回的bodyEOF=false导致永远不会执行tryPutIdleConn(trace)。
//注意pc.sawEOF和body Close没关系，这个是检测conn是否Close
alive = alive &&
bodyEOF &&
!pc.sawEOF &&
pc.wroteRequest() &&
replaced && tryPutIdleConn(trace)
if bodyEOF {
eofc <- struct{}{}
}
case <-rc.req.Cancel:
alive = false
pc.t.CancelRequest(rc.req)
case <-rc.req.Context().Done():
alive = false
pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err())
case <-pc.closech:
alive = false
}
testHookReadLoopBeforeNextRead()
}
}下面我们来看下resp.Body的结构bodyEOFSignal:
//readLoop()里面的body
body := &bodyEOFSignal{
body: resp.Body,
//如果调用earlyCloseFn就执行waitForBodyRead <- false
earlyCloseFn: func() error {
waitForBodyRead <- false
<-eofc // will be closed by deferred call at the end of the function
return nil
},
//如果调用fn()并且 err == io.EOF 就执行waitForBodyRead <- true
fn: func(err error) error {
isEOF := err == io.EOF
waitForBodyRead <- isEOF
if isEOF {
<-eofc // see comment above eofc declaration
} else if err != nil {
if cerr := pc.canceled(); cerr != nil {
return cerr
}
}
return err
},
}
// bodyEOFSignal is used by the HTTP/1 transport when reading response
// bodies to make sure we see the end of a response body before
// proceeding and reading on the connection again.
//
// It wraps a ReadCloser but runs fn (if non-nil) at most
// once, right before its final (error-producing) Read or Close call
// returns. fn should return the new error to return from Read or Close.
//
// If earlyCloseFn is non-nil and Close is called before io.EOF is
// seen, earlyCloseFn is called instead of fn, and its return value is
// the return value from Close.
type bodyEOFSignal struct {
body io.ReadCloser
mu sync.Mutex // guards following 4 fields
closed bool // whether Close has been called
rerr error // sticky Read error
fn func(error) error // err will be nil on Read io.EOF
earlyCloseFn func() error // optional alt Close func used if io.EOF not seen 如果没EOF就会被调用
}
func (es *bodyEOFSignal) Read(p []byte) (n int, err error) {
es.mu.Lock()
closed, rerr := es.closed, es.rerr
es.mu.Unlock()
if closed {
return 0, errReadOnClosedResBody
}
if rerr != nil {
return 0, rerr
}
n, err = es.body.Read(p)
if err != nil {
es.mu.Lock()
defer es.mu.Unlock()
if es.rerr == nil {
es.rerr = err
}
//condfn会去执行es.fn
err = es.condfn(err)
}
return
}
func (es *bodyEOFSignal) Close() error {
es.mu.Lock()
defer es.mu.Unlock()
if es.closed {
return nil
}
es.closed = true
// 如果调用Close的时候没有EOF就会调用earlyCloseFn()
if es.earlyCloseFn != nil && es.rerr != io.EOF {
return es.earlyCloseFn()
}
err := es.body.Close()
return es.condfn(err)
}
// caller must hold es.mu.
// 把err传给es.fn, es.fn会通过err做不同的操作，主要是判断err是不是EOF
func (es *bodyEOFSignal) condfn(err error) error {
if es.fn == nil {
return err
}
err = es.fn(err)
es.fn = nil
return err
}总结
我们收到的resp.body是bodyEOFSignal,如果不执行Close()会导致readLoop()阻塞。此时如果设置了最大连接MaxConnsPerHost则达到连接数后不能再申请。但是如果只是Close()而不执行Read()则会导致连接不会放回idleConn
推荐用法:
不一定要执行Close(),但一定要执行Read():
如不需要body里面的内容可以执行
client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer io.Copy(io.Discard,resp.Body)如果需要resp.Body则执行:
client := http.DefaultClient
resp, err := client.Do(req)
if err != nil {
return nil, err
}
//这里的close是防止Read操作不能被正确执行到做兜底,
//如果能够确保resp.Body被执行到也不需要Close
defer resp.Body.Close()
//读取resp.Body,如io.ReadAll/io.Copy()...
io.ReadAll(resp.Body)