【工作笔记】Http响应头过长

起因

突然有测试小伙伴反馈进公司官网主页会白屏，但只是个例不是普遍现象
查监控发现没监控到异常问题
查了很久（这个很久单指对于线上问题来说）才定位是请求的异常，因为这套系统的异常用的是 @ExceptionHandler，这也导致问题排查多绕了不少圈子

原因

异常：org.springframework.web.util.NestedServletException : Request processing failed; nested exception is org.apache.coyote.http11.HeadersTooLargeException: An attempt was made to write more data to the response headers than there was room available in the buffer. Increase maxHttpHeaderSize on the connector or write less data into the response headers.
	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:982)
	at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:634)
	at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:741)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:231)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.boot.web.filter.ApplicationContextHeaderFilter.doFilterInternal(ApplicationContextHeaderFilter.java:55)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:81)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.session.web.http.SessionRepositoryFilter.doFilterInternal(SessionRepositoryFilter.java:167)
	at org.springframework.session.web.http.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:80)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.cloud.sleuth.instrument.web.TraceFilter.doFilter(TraceFilter.java:150)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.springframework.boot.actuate.autoconfigure.MetricsFilter.doFilterInternal(MetricsFilter.java:106)
	at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
	at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
	at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
	at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:199)
	at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
	at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:543)
	at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
	at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:81)
	at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87)
	at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
	at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:609)
	at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
	at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:818)
	at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1623)
	at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.coyote.http11.HeadersTooLargeException : An attempt was made to write more data to the response headers than there was room available in the buffer. Increase maxHttpHeaderSize on the connector or write less data into the response headers.
	at org.apache.coyote.http11.Http11OutputBuffer.checkLengthBeforeWrite(Http11OutputBuffer.java:543)
	at org.apache.coyote.http11.Http11OutputBuffer.write(Http11OutputBuffer.java:468)
	at org.apache.coyote.http11.Http11OutputBuffer.write(Http11OutputBuffer.java:454)
	at org.apache.coyote.http11.Http11OutputBuffer.sendHeader(Http11OutputBuffer.java:414)
	at org.apache.coyote.http11.Http11Processor.prepareResponse(Http11Processor.java:1200)
	at org.apache.coyote.AbstractProcessor.action(AbstractProcessor.java:380)
	at org.apache.coyote.Response.action(Response.java:208)
	at org.apache.coyote.Response.sendHeaders(Response.java:418)
	at org.apache.catalina.connector.OutputBuffer.doFlush(OutputBuffer.java:312)
	at org.apache.catalina.connector.OutputBuffer.flush(OutputBuffer.java:293)
	at org.apache.catalina.connector.CoyoteOutputStream.flush(CoyoteOutputStream.java:118)
	at org.springframework.cloud.sleuth.instrument.web.TraceServletOutputStream.flush(TraceServletOutputStream.java:128)
	at org.springframework.session.web.http.OnCommittedResponseWrapper$SaveContextServletOutputStream.flush(OnCommittedResponseWrapper.java:455)
	at com.fasterxml.jackson.core.json.UTF8JsonGenerator.flush(UTF8JsonGenerator.java:1054)
	at com.fasterxml.jackson.databind.ObjectWriter.writeValue(ObjectWriter.java:953)
	at org.springframework.http.converter.json.AbstractJackson2HttpMessageConverter.writeInternal(AbstractJackson2HttpMessageConverter.java:286)
	at org.springframework.http.converter.AbstractGenericHttpMessageConverter.write(AbstractGenericHttpMessageConverter.java:106)
	at org.springframework.web.servlet.mvc.method.annotation.AbstractMessageConverterMethodProcessor.writeWithMessageConverters(AbstractMessageConverterMethodProcessor.java:231)
	at org.springframework.web.servlet.mvc.method.annotation.RequestResponseBodyMethodProcessor.handleReturnValue(RequestResponseBodyMethodProcessor.java:174)
	at org.springframework.web.method.support.HandlerMethodReturnValueHandlerComposite.handleReturnValue(HandlerMethodReturnValueHandlerComposite.java:81)
	at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:113)
	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827)
	at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738)
	at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85)
	at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:967)
	at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:901)
	at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970)
	... 90 more

第一次修复

不就是响应头嘛，先紧急修复下，直接无限大，再说。。。

server:
  max-http-header-size: -1

第二次修复

改完紧急发布发现没生效。。。不用说肯定掉坑里了。。。看看代码才发现，-1 这个无限大是 tomcat 的玩法，我 Spring 可不惯着你
Spring 底层是根据是不是为 0 做更新的。。

		void customizeTomcat(ServerProperties serverProperties,
				TomcatEmbeddedServletContainerFactory factory) {
			if (getBasedir() != null) {
				factory.setBaseDirectory(getBasedir());
			}
			factory.setBackgroundProcessorDelay(Tomcat.this.backgroundProcessorDelay);
			customizeRemoteIpValve(serverProperties, factory);
			if (this.maxThreads > 0) {
				customizeMaxThreads(factory);
			}
			if (this.minSpareThreads > 0) {
				customizeMinThreads(factory);
			}
			// 老6了吧
			int maxHttpHeaderSize = (serverProperties.getMaxHttpHeaderSize() > 0
					? serverProperties.getMaxHttpHeaderSize() : this.maxHttpHeaderSize);
			if (maxHttpHeaderSize > 0) {
				customizeMaxHttpHeaderSize(factory, maxHttpHeaderSize);
			}
			if (this.maxHttpPostSize != 0) {
				customizeMaxHttpPostSize(factory, this.maxHttpPostSize);
			}
			if (this.accesslog.enabled) {
				customizeAccessLog(factory);
			}
			if (getUriEncoding() != null) {
				factory.setUriEncoding(getUriEncoding());
			}
			if (serverProperties.getConnectionTimeout() != null) {
				customizeConnectionTimeout(factory,
						serverProperties.getConnectionTimeout());
			}
			if (this.redirectContextRoot != null) {
				customizeRedirectContextRoot(factory, this.redirectContextRoot);
			}
			if (this.maxConnections > 0) {
				customizeMaxConnections(factory);
			}
			if (this.acceptCount > 0) {
				customizeAcceptCount(factory);
			}
			if (!ObjectUtils.isEmpty(this.additionalTldSkipPatterns)) {
				factory.getTldSkipPatterns().addAll(this.additionalTldSkipPatterns);
			}
		}

server:
  #  Maximum size in bytes of the HTTP message header
  max-http-header-size: 1048576

彻底修复

紧急处理完，开始正式挖掘问题处理
前后端大佬仔细排查去除了三个前端监控组件，彻底解决问题

一个小记录

其实我就是想记录下这个类：org.springframework.boot.autoconfigure.web.ServerProperties
工作中用的最多缺一直忽视的配置类。。。线上不少关于请求的异常都可以在这里找到配置处理
类似的：上传文件大小、响应头大小、请求数等等