原文:
https://medium.com/@owlwalks/sending-big-file-with-minimal-memory-in-golang-8f3fc280d2c
一般我们发送文件
buf := new(bytes.Buffer)
writer := multipart.NewWriter(buf)
defer writer.Close()
part, err := writer.CreateFormFile("myFile", "foo.txt")
if err != nil {
return err
}
file, err := os.Open(name)
if err != nil {
return err
}
defer file.Close()
if _, err = io.Copy(part, file); err != nil {
return err
}
http.Post(url, writer.FormDataContentType(), buf)
这样buf会读取文件的所有内容,加入文件非常大,内存占用就会比较大
优化方法
r, w := io.Pipe()
m := multipart.NewWriter(w)
go func() {
defer w.Close()
defer m.Close()
part, err := m.CreateFormFile("myFile", "foo.txt")
if err != nil {
return
}
file, err := os.Open(name)
if err != nil {
return
}
defer file.Close()
if _, err = io.Copy(part, file); err != nil {
return
}
}()
http.Post(url, m.FormDataContentType(), r)
上述是代码是从原处拷贝,下面分析下原因
net/http 中
func Post(url, contentType string, body io.Reader) (resp *Response, err error) {
] return DefaultClient.Post(url, contentType, body)
]}
func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, err error) {
req, err := NewRequest("POST", url, body)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", contentType)
return c.Do(req)
}
func NewRequest(method, url string, body io.Reader) (*Request, error) {
...
if body != nil {
switch v := body.(type) {
case *bytes.Buffer:
req.ContentLength = int64(v.Len())
buf := v.Bytes()
req.GetBody = func() (io.ReadCloser, error) {
r := bytes.NewReader(buf)
return ioutil.NopCloser(r), nil
}
case *bytes.Reader:
req.ContentLength = int64(v.Len())
snapshot := *v
req.GetBody = func() (io.ReadCloser, error) {
r := snapshot
return ioutil.NopCloser(&r), nil
}
case *strings.Reader:
req.ContentLength = int64(v.Len())
snapshot := *v
req.GetBody = func() (io.ReadCloser, error) {
r := snapshot
return ioutil.NopCloser(&r), nil
}
default:
// This is where we'd set it to -1 (at least
// if body != NoBody) to mean unknown, but
// that broke people during the Go 1.8 testing
// period. People depend on it being 0 I
// guess. Maybe retry later. See Issue 18117.
}
...
}
os中
func Pipe() (r *File, w *File, err error) {
var p [2]int
e := syscall.Pipe2(p[0:], syscall.O_CLOEXEC)
// pipe2 was added in 2.6.27 and our minimum requirement is 2.6.23, so it
// might not be implemented.
if e == syscall.ENOSYS {
// See ../syscall/exec.go for description of lock.
syscall.ForkLock.RLock()
e = syscall.Pipe(p[0:])
if e != nil {
syscall.ForkLock.RUnlock()
return nil, nil, NewSyscallError("pipe", e)
}
syscall.CloseOnExec(p[0])
syscall.CloseOnExec(p[1])
syscall.ForkLock.RUnlock()
} else if e != nil {
return nil, nil, NewSyscallError("pipe2", e)
}
return newFile(uintptr(p[0]), "|0", kindPipe), newFile(uintptr(p[1]), "|1", kindPipe), nil
}
可见Pipe返回的类型在body的类型判断中进入了default的逻辑,而追溯post的方法会在此处写
func (t *transferWriter) writeBody(w io.Writer) error {
...
if t.Body != nil {
var body = transferBodyReader{t}
if chunked(t.TransferEncoding) {
if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse {
w = &internal.FlushAfterChunkWriter{Writer: bw}
}
cw := internal.NewChunkedWriter(w)
_, err = io.Copy(cw, body)
if err == nil {
err = cw.Close()
}
} else if t.ContentLength == -1 {
ncopy, err = io.Copy(w, body)
} else {
ncopy, err = io.Copy(w, io.LimitReader(body, t.ContentLength))
if err != nil {
return err
}
var nextra int64
nextra, err = io.Copy(ioutil.Discard, body)
ncopy += nextra
}
...
}
由于body不为nil,而且contentlength为0,所以进入了else的逻辑,也就形成了流式读取和流式写入,在大文件时候可以节省内存