go标准库httputil.ReverseProxy简单介绍和使用避坑

很久没水博客了，今天就来水一篇，说说go标准库的httputil.ReverseProxy
httputil.ReverseProxy顾名思义，http的反向代理，可以类比nginx的反向代理功能

type ReverseProxy struct {
	// Director must be a function which modifies
	// the request into a new request to be sent
	// using Transport. Its response is then copied
	// back to the original client unmodified.
	// Director must not access the provided Request
	// after returning.
	Director func(*http.Request)

	// The transport used to perform proxy requests.
	// If nil, http.DefaultTransport is used.
	Transport http.RoundTripper

	// FlushInterval specifies the flush interval
	// to flush to the client while copying the
	// response body.
	// If zero, no periodic flushing is done.
	// A negative value means to flush immediately
	// after each write to the client.
	// The FlushInterval is ignored when ReverseProxy
	// recognizes a response as a streaming response, or
	// if its ContentLength is -1; for such responses, writes
	// are flushed to the client immediately.
	FlushInterval time.Duration

	// ErrorLog specifies an optional logger for errors
	// that occur when attempting to proxy the request.
	// If nil, logging is done via the log package's standard logger.
	ErrorLog *log.Logger

	// BufferPool optionally specifies a buffer pool to
	// get byte slices for use by io.CopyBuffer when
	// copying HTTP response bodies.
	BufferPool BufferPool

	// ModifyResponse is an optional function that modifies the
	// Response from the backend. It is called if the backend
	// returns a response at all, with any HTTP status code.
	// If the backend is unreachable, the optional ErrorHandler is
	// called without any call to ModifyResponse.
	//
	// If ModifyResponse returns an error, ErrorHandler is called
	// with its error value. If ErrorHandler is nil, its default
	// implementation is used.
	ModifyResponse func(*http.Response) error

	// ErrorHandler is an optional function that handles errors
	// reaching the backend or errors from ModifyResponse.
	//
	// If nil, the default is to log the provided error and return
	// a 502 Status Bad Gateway response.
	ErrorHandler func(http.ResponseWriter, *http.Request, error)
}

httputil.ReverseProxy 定义了一组方法让使用者去实现，主要有这几个

• Director
  最核心的方法， 我们可以在这里对请求进行相应的修改，比如设置请求目标的地址，对原有请求头进行增删改，以及对请求体进行处理等等操作。

• ModifyResponse
  可以让我们对响应的结果进行处理，比如修改、读取响应头和响应体。

• ErrorHandler
  请求出错或者ModifyResponse返回error时会回调该方法，比如目标服务器无法连接，请求超时等等

看一个简单的示例代码

func main() {

	proxy := &httputil.ReverseProxy{

		Director: func(req *http.Request) {
			u, _ := url.Parse("https://www.qq.com/")
			req.URL = u
			req.Host = u.Host // 必须显示修改Host，否则转发可能失败
		},

		ModifyResponse: func(resp *http.Response) error {
			log.Println("resp status:", resp.Status)
			log.Println("resp headers:")
			for hk, hv := range resp.Header {
				log.Println(hk, ":", strings.Join(hv, ","))
			}
			return nil
		},

		ErrorLog: log.New(os.Stdout, "ReverseProxy:", log.LstdFlags | log.Lshortfile),

		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
			if err != nil {
				log.Println("ErrorHandler catch err:", err)
				
				w.WriteHeader(http.StatusBadGateway)
				_, _ = fmt.Fprintf(w, err.Error())
			}
		},
	}

	http.Handle("/", proxy)

	if err := http.ListenAndServe(":1111", nil); err != nil {
		log.Fatal(err)
	}
}

访问效果

req.Host = u.Host 这句代码千万不要忘记，否则可能导致代理访问失败，因为在golang中修改http.Request.URL.Host并不等同于Host请求头，除非本身Host请求头是空的，这时会使用URL.Host的值。

摘录URL.Host 和 Request.Host 解释

// For client requests, the URL's Host specifies the server to
// connect to, while the Request's Host field optionally
// specifies the Host header value to send in the HTTP
// request.

// For client requests, Host optionally overrides the Host
// header to send. If empty, the Request.Write method uses
// the value of URL.Host. Host may contain an international
// domain name.

不修改 Request.Host 请求失败：

可以简单写个tcp服务查看经代理转发后的请求数据

func main() {

	listener, err := net.Listen("tcp4", ":1112")

	if err != nil {
		log.Fatal(err)
	}

	for {
		conn, err := listener.Accept()

		if err != nil {
			log.Println("accept err:", err)
			continue
		}

		// 另开一个协程处理tcp连接
		go func(conn net.Conn) {
			defer conn.Close()

			log.Println("build new connection")

			buf := make([]byte, 4096)
			sb := &strings.Builder{}

			for {
				n, err := conn.Read(buf)
				if err != nil {
					if errors.Is(err, os.ErrDeadlineExceeded) {
						log.Println("read timeout")
						break
					}

					if errors.Is(err, io.EOF) {
						log.Println("read eof")
						break
					}

					log.Println("read from client err:", err)
					return
				}
				sb.Write(buf[:n])
				if err := conn.SetReadDeadline(time.Now().Add(time.Millisecond * 10)); err != nil {
					log.Println("setTimeout err:", err)
					return
				}
			}

			log.Println(sb.String())

			if err := conn.SetWriteDeadline(time.Now().Add(time.Second * 3)); err != nil {
				log.Println("setTimeout err:", err)
				return
			}
			conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: Closed\r\n\r\nOK"))

		}(conn)
	}
}

设置代理访问上述服务 并注释掉 req.Host 设置代码，观察tcp服务日志输出：

func main() {

	proxy := &httputil.ReverseProxy{

		Director: func(req *http.Request) {
			u, _ := url.Parse("http://localhost:1112")
			req.URL = u
			//req.Host = u.Host // 必须显示修改Host，否则转发可能失败
		},

		ModifyResponse: func(resp *http.Response) error {
			log.Println("resp status:", resp.Status)
			log.Println("resp headers:")
			for hk, hv := range resp.Header {
				log.Println(hk, ":", strings.Join(hv, ","))
			}
			return nil
		},

		ErrorLog: log.New(os.Stdout, "ReverseProxy:", log.LstdFlags | log.Lshortfile),

		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
			if err != nil {
				log.Println("ErrorHandler catch err:", err)
				
				w.WriteHeader(http.StatusBadGateway)
				_, _ = fmt.Fprintf(w, err.Error())
			}
		},
	}

	http.Handle("/", proxy)

	if err := http.ListenAndServe(":1111", nil); err != nil {
		log.Fatal(err)
	}
}

2022/06/12 18:32:12 build new connection
2022/06/12 18:32:12 read timeout
2022/06/12 18:32:12 GET / HTTP/1.1
Host: 127.0.0.1:1111
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36
Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
Accept-Encoding: gzip, deflate, br
Accept-Language: zh-CN,zh;q=0.9
Cache-Control: max-age=0
Cookie: ts_uid=2693745378; ariaDefaultTheme=undefined; ad_play_index=85
Sec-Ch-Ua: " Not A;Brand";v="99", "Chromium";v="102", "Google Chrome";v="102"
Sec-Ch-Ua-Mobile: ?0
Sec-Ch-Ua-Platform: "Windows"
Sec-Fetch-Dest: document
Sec-Fetch-Mode: navigate
Sec-Fetch-Site: none
Sec-Fetch-User: ?1
Upgrade-Insecure-Requests: 1
X-Forwarded-For: 127.0.0.1

转发后请求的地址是 localhost:1112，但Host还是原始请求的Host，而http服务器通常需要通过Host来区分不同的虚拟服务器，从而出现异常。